From b69c7cc4db3a18f4fed1dbd7d5c441f514c2d516 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:02:22 +0200 Subject: [PATCH 001/109] feat(components): generate the native Reflux v1 contract --- ...TION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md | 23 +++-- .../config/generated_component_abi.hpp | 98 ++++++++++++++++++- .../config/generated_component_catalog.hpp | 6 +- .../config/generated_route_accessors.inc | 2 +- .../init/generated_component_invokers.inc | 2 +- .../pops/_generated_component_interfaces.py | 12 ++- python/pops/interfaces.py | 3 +- .../pops/model/_generated_component_schema.py | 4 +- .../runtime/_generated_component_routes.py | 6 +- schemas/component_catalog.v2.json | 10 ++ scripts/generate_component_catalog.py | 60 ++++++++++++ 11 files changed, 206 insertions(+), 20 deletions(-) diff --git a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md index e8bb1baf1..f7652d26d 100644 --- a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md +++ b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md @@ -705,9 +705,13 @@ doivent couvrir exactement la hiérarchie. Le provider natif livré matérialise le coeur maillage/stockage en 2D et ses kernels de transfert, correction conservative et sous-cyclage AMR exigent un ratio de transition égal à 2. La correction -coarse/fine reste l'unique ledger de flux détenu par PoPS : aucune interface externe `Reflux` -n'existe, car déléguer ce dépôt créerait une seconde autorité conservative. Une autre dimension ou un autre -ratio est refusé pendant la résolution ou le bind avec les capacités observées. Le coeur de +coarse/fine reste l'unique ledger de flux détenu par PoPS. L'interface native `Reflux` ne peut +déléguer qu'un kernel local et non collectif : PoPS lui fournit les flux coarse/fine déjà intégrés +dans le temps et ramenés sur la même face coarse ; le kernel écrit la correction locale +`side * (fine - coarse) / dx`. PoPS conserve exclusivement la topologie d'interface, le ledger, la +réduction MPI, la transaction et l'application à l'état. Un provider `Reflux` ne devient donc jamais +une seconde autorité conservative. Une autre dimension ou un autre ratio est refusé pendant la +résolution ou le bind avec les capacités observées. Le coeur de planification ne normalise jamais la demande vers ce sous-ensemble. Défensivement, `AmrProgramContext` revalide aussi chaque transition à sa construction et refuse un ratio différent de 2 avant le premier pas : cette limite appartient au provider natif reflux/average-down installé, @@ -1405,14 +1409,21 @@ paramètres, interfaces, requirements, capabilities, effets, layouts, clocks, d restart et points d'entrée. Le même catalogue génère les IDs et tables C/POD versionnées des interfaces natives (flux numérique, -ghost boundary, closure de champ, tagging, clustering, transfert, solveur de champ, writer et -topologie de champ). Le reflux conservatif reste une autorité interne pilotée par le flux ledger ; -aucune table externe `Reflux` n'est annoncée. Chaque famille possède sa propre version d'interface, indépendante de la version +ghost boundary, closure de champ, tagging, clustering, transfert, kernel local de reflux, solveur de +champ, writer et topologie de champ). Le reflux conservatif complet reste une autorité interne +pilotée par le flux ledger ; la table externe `Reflux` ne couvre que la transformation locale, +non collective, de flux intégrés en correction non appliquée. Chaque famille possède sa propre version d'interface, indépendante de la version du protocole enveloppe. Le loader authentifie identité sémantique, manifest, digest du catalogue, taille/header de table et opérations requises avant de conserver le handle de bibliothèque. Les tables sont résolues une fois à l'installation ; aucun `dlsym`, nom de classe ou dispatch Python n'entre dans une boucle de cellules. +Le contrat `Reflux` v1 est volontairement livré avant son branchement dans +`PreparedAmrProgramRefluxTransition` : catalogue, manifest, loader et consumer typé peuvent qualifier +un conformer, mais le runtime AMR continue d'utiliser son kernel interne tant qu'un adaptateur préparé +ne peut pas fournir les vues locales sans dupliquer le ledger ni transférer l'autorité collective. Une +configuration AMR ne prétend donc pas encore avoir sélectionné un provider `Reflux` externe. + Les champs sémantiques inconnus, capacités sans preuve, collisions d'identité et entry points manquants sont refusés. Un vieux manifest n'est pas « réparé » silencieusement. diff --git a/include/pops/runtime/config/generated_component_abi.hpp b/include/pops/runtime/config/generated_component_abi.hpp index 2403ce736..bc042a6f9 100644 --- a/include/pops/runtime/config/generated_component_abi.hpp +++ b/include/pops/runtime/config/generated_component_abi.hpp @@ -15,7 +15,7 @@ extern "C" { #endif #define POPS_COMPONENT_API_SYMBOL_V1 "pops_component_interface_v1" -#define POPS_COMPONENT_CATALOG_SHA256_V1 "84c68fcee96663f71e7e7fa7589ec1ddee0d1037e741a678e4afd83c9749620a" +#define POPS_COMPONENT_CATALOG_SHA256_V1 "5c991781aadafd3165dccb4642086c8b20fbd4a83ee8e462f1e29078ecb0d1c4" #define POPS_COMPONENT_PROTOCOL_ABI_V1 1u #define POPS_COMPONENT_COMMON_ABI_V1 1u @@ -26,6 +26,7 @@ typedef enum PopsNativeInterfaceIdV1 { POPS_NATIVE_INTERFACE_TAGGER_V2 = 3, POPS_NATIVE_INTERFACE_CLUSTERING_V1 = 4, POPS_NATIVE_INTERFACE_TRANSFER_V1 = 5, + POPS_NATIVE_INTERFACE_REFLUX_V1 = 6, POPS_NATIVE_INTERFACE_FIELD_SOLVER_V2 = 7, POPS_NATIVE_INTERFACE_WRITER_V1 = 8, POPS_NATIVE_INTERFACE_FIELD_TOPOLOGY_V2 = 9, @@ -446,6 +447,41 @@ typedef struct PopsTransferApiV1 { PopsTransferApplyFnV1 apply; } PopsTransferApiV1; +// Reflux providers are patch-local numerical kernels only. PoPS retains sole ownership of the +// time-integrated flux ledger, interface topology, MPI reduction, transaction and state update. +// Each face contains coarse/fine fluxes already integrated in time and averaged onto the same +// coarse face. The provider writes, but never applies, side*(fine-coarse)/dx into `correction`. +typedef enum PopsRefluxFaceSideV1 { + POPS_REFLUX_FACE_LOW_V1 = -1, + POPS_REFLUX_FACE_HIGH_V1 = 1 +} PopsRefluxFaceSideV1; +typedef struct PopsRefluxFaceV1 { + uint32_t struct_size; + const char* interface_identity; + int32_t axis; + PopsRefluxFaceSideV1 side; + double inverse_coarse_cell_spacing; + PopsConstFieldViewV1 coarse_integrated_flux; + PopsConstFieldViewV1 fine_integrated_flux; + PopsFieldViewV1 correction; +} PopsRefluxFaceV1; +typedef struct PopsRefluxRequestV1 { + uint32_t struct_size; + const char* transition_identity; + int32_t parent_level; + int32_t child_level; + size_t face_count; + const PopsRefluxFaceV1* faces; + PopsLogicalTimeV1 logical_time; + PopsExecutionContextV1 execution; +} PopsRefluxRequestV1; +typedef int32_t (*PopsRefluxApplyInterfaceBatchFnV1)( + void*, const PopsRefluxRequestV1*, PopsComponentStatusV1*); +typedef struct PopsRefluxApiV1 { + PopsComponentTableHeaderV1 header; + PopsRefluxApplyInterfaceBatchFnV1 apply_interface_batch; +} PopsRefluxApiV1; + typedef struct PopsFieldPatchMetadataV1 { uint32_t struct_size; size_t global_patch_index; @@ -724,6 +760,7 @@ inline constexpr size_t generated_native_interface_table_size( case POPS_NATIVE_INTERFACE_TAGGER_V2: return sizeof(PopsTaggerApiV2); case POPS_NATIVE_INTERFACE_CLUSTERING_V1: return sizeof(PopsClusteringApiV1); case POPS_NATIVE_INTERFACE_TRANSFER_V1: return sizeof(PopsTransferApiV1); + case POPS_NATIVE_INTERFACE_REFLUX_V1: return sizeof(PopsRefluxApiV1); case POPS_NATIVE_INTERFACE_FIELD_SOLVER_V2: return sizeof(PopsFieldSolverApiV2); case POPS_NATIVE_INTERFACE_WRITER_V1: return sizeof(PopsWriterApiV1); case POPS_NATIVE_INTERFACE_FIELD_TOPOLOGY_V2: return sizeof(PopsFieldTopologyApiV2); @@ -739,12 +776,71 @@ inline constexpr const char* generated_native_interface_table_name( case POPS_NATIVE_INTERFACE_TAGGER_V2: return "PopsTaggerApiV2"; case POPS_NATIVE_INTERFACE_CLUSTERING_V1: return "PopsClusteringApiV1"; case POPS_NATIVE_INTERFACE_TRANSFER_V1: return "PopsTransferApiV1"; + case POPS_NATIVE_INTERFACE_REFLUX_V1: return "PopsRefluxApiV1"; case POPS_NATIVE_INTERFACE_FIELD_SOLVER_V2: return "PopsFieldSolverApiV2"; case POPS_NATIVE_INTERFACE_WRITER_V1: return "PopsWriterApiV1"; case POPS_NATIVE_INTERFACE_FIELD_TOPOLOGY_V2: return "PopsFieldTopologyApiV2"; } return nullptr; } +inline bool generated_native_interface_table_is_complete( + PopsNativeInterfaceIdV1 id, const void* table, size_t table_size) noexcept { + if (table == nullptr) + return false; + switch (id) { + case POPS_NATIVE_INTERFACE_NUMERICAL_FLUX_V1: { + if (table_size < sizeof(PopsNumericalFluxApiV1)) return false; + const auto* api = static_cast(table); + return api->evaluate_faces != nullptr; + } + case POPS_NATIVE_INTERFACE_GHOST_BOUNDARY_V1: { + if (table_size < sizeof(PopsGhostBoundaryApiV1)) return false; + const auto* api = static_cast(table); + return api->apply_region_batch != nullptr; + } + case POPS_NATIVE_INTERFACE_FIELD_BOUNDARY_CLOSURE_V1: { + if (table_size < sizeof(PopsFieldBoundaryClosureApiV1)) return false; + const auto* api = static_cast(table); + return api->residual != nullptr && api->jvp != nullptr; + } + case POPS_NATIVE_INTERFACE_TAGGER_V2: { + if (table_size < sizeof(PopsTaggerApiV2)) return false; + const auto* api = static_cast(table); + return api->tag_batch != nullptr; + } + case POPS_NATIVE_INTERFACE_CLUSTERING_V1: { + if (table_size < sizeof(PopsClusteringApiV1)) return false; + const auto* api = static_cast(table); + return api->cluster != nullptr; + } + case POPS_NATIVE_INTERFACE_TRANSFER_V1: { + if (table_size < sizeof(PopsTransferApiV1)) return false; + const auto* api = static_cast(table); + return api->apply != nullptr; + } + case POPS_NATIVE_INTERFACE_REFLUX_V1: { + if (table_size < sizeof(PopsRefluxApiV1)) return false; + const auto* api = static_cast(table); + return api->apply_interface_batch != nullptr; + } + case POPS_NATIVE_INTERFACE_FIELD_SOLVER_V2: { + if (table_size < sizeof(PopsFieldSolverApiV2)) return false; + const auto* api = static_cast(table); + return api->solve != nullptr; + } + case POPS_NATIVE_INTERFACE_WRITER_V1: { + if (table_size < sizeof(PopsWriterApiV1)) return false; + const auto* api = static_cast(table); + return api->verify != nullptr && api->publish != nullptr && api->discard != nullptr && api->rollback != nullptr; + } + case POPS_NATIVE_INTERFACE_FIELD_TOPOLOGY_V2: { + if (table_size < sizeof(PopsFieldTopologyApiV2)) return false; + const auto* api = static_cast(table); + return api->prepare_topology != nullptr; + } + } + return false; +} } // namespace pops::component #endif // clang-format on diff --git a/include/pops/runtime/config/generated_component_catalog.hpp b/include/pops/runtime/config/generated_component_catalog.hpp index 2c4cb1c08..b1c996aa2 100644 --- a/include/pops/runtime/config/generated_component_catalog.hpp +++ b/include/pops/runtime/config/generated_component_catalog.hpp @@ -301,9 +301,9 @@ inline constexpr int kComponentCatalogSchemaVersion = 1; inline constexpr int kComponentManifestSchemaVersion = 2; inline constexpr int kRouteRegistryVersion = 2; inline constexpr int kCapabilityVocabularyVersion = 2; -inline constexpr const char* kComponentCatalogSha256 = "84c68fcee96663f71e7e7fa7589ec1ddee0d1037e741a678e4afd83c9749620a"; -inline constexpr const char* kComponentCatalogSemanticSha256 = "c0e14d4a3dd082612d052b2ce293e17de712c6916e39f430d0ff311ed0f24ef8"; -inline constexpr const char* kRouteRegistrySignature = "v2:c0e14d4a3dd082612d052b2ce293e17de712c6916e39f430d0ff311ed0f24ef8"; +inline constexpr const char* kComponentCatalogSha256 = "5c991781aadafd3165dccb4642086c8b20fbd4a83ee8e462f1e29078ecb0d1c4"; +inline constexpr const char* kComponentCatalogSemanticSha256 = "b29e5d1d811234bd55bfce83ffdf810ed95868392d1ac1da5644f8886620e129"; +inline constexpr const char* kRouteRegistrySignature = "v2:b29e5d1d811234bd55bfce83ffdf810ed95868392d1ac1da5644f8886620e129"; inline constexpr const char* kComponentManifestSemanticFields[] = { "schema_version", "uri", diff --git a/include/pops/runtime/config/generated_route_accessors.inc b/include/pops/runtime/config/generated_route_accessors.inc index 886506256..8d847fc57 100644 --- a/include/pops/runtime/config/generated_route_accessors.inc +++ b/include/pops/runtime/config/generated_route_accessors.inc @@ -1,4 +1,4 @@ -// Generated from component catalog 84c68fcee96663f71e7e7fa7589ec1ddee0d1037e741a678e4afd83c9749620a; DO NOT EDIT. +// Generated from component catalog 5c991781aadafd3165dccb4642086c8b20fbd4a83ee8e462f1e29078ecb0d1c4; DO NOT EDIT. // POPS_DEFINE_ROUTE_ACCESSORS must be defined by the including behavior header. POPS_DEFINE_ROUTE_ACCESSORS(riemann, RiemannRouteId, kRiemannRoutes, kRiemann); POPS_DEFINE_ROUTE_ACCESSORS(limiter, LimiterRouteId, kLimiterRoutes, kLimiter); diff --git a/python/bindings/core/init/generated_component_invokers.inc b/python/bindings/core/init/generated_component_invokers.inc index cc1600dc8..d9ddca9d6 100644 --- a/python/bindings/core/init/generated_component_invokers.inc +++ b/python/bindings/core/init/generated_component_invokers.inc @@ -1,4 +1,4 @@ -// Generated by scripts/generate_component_catalog.py from catalog 84c68fcee96663f71e7e7fa7589ec1ddee0d1037e741a678e4afd83c9749620a; DO NOT EDIT. +// Generated by scripts/generate_component_catalog.py from catalog 5c991781aadafd3165dccb4642086c8b20fbd4a83ee8e462f1e29078ecb0d1c4; DO NOT EDIT. // This file is the sole Python/native request marshaller. init_component_loader.cpp only registers it. #include diff --git a/python/pops/_generated_component_interfaces.py b/python/pops/_generated_component_interfaces.py index 66d02552c..2e99f6ef2 100644 --- a/python/pops/_generated_component_interfaces.py +++ b/python/pops/_generated_component_interfaces.py @@ -3,8 +3,8 @@ NATIVE_COMPONENT_ABI_VERSION = 1 NATIVE_COMPONENT_COMMON_ABI_VERSION = 1 -NATIVE_COMPONENT_CATALOG_SHA256 = '84c68fcee96663f71e7e7fa7589ec1ddee0d1037e741a678e4afd83c9749620a' -NATIVE_COMPONENT_CATALOG_SEMANTIC_SHA256 = 'c0e14d4a3dd082612d052b2ce293e17de712c6916e39f430d0ff311ed0f24ef8' +NATIVE_COMPONENT_CATALOG_SHA256 = '5c991781aadafd3165dccb4642086c8b20fbd4a83ee8e462f1e29078ecb0d1c4' +NATIVE_COMPONENT_CATALOG_SEMANTIC_SHA256 = 'b29e5d1d811234bd55bfce83ffdf810ed95868392d1ac1da5644f8886620e129' NATIVE_TAGGING_PROGRAM_ABI = {'version': 1, 'execution_modes': {'native_backend': 1, 'host': 2}, 'collective_scopes': {'none': 0}, @@ -72,6 +72,14 @@ 'hot_path': True, 'facets': ('stencil', 'lowering'), 'operations': ('apply',)}, + {'id': 6, + 'name': 'reflux', + 'uri': 'pops://interfaces/reflux', + 'version': 1, + 'cpp_table': 'PopsRefluxApiV1', + 'hot_path': True, + 'facets': ('stencil', 'lowering', 'effects'), + 'operations': ('apply_interface_batch',)}, {'id': 7, 'name': 'field_solver', 'uri': 'pops://interfaces/field-solver', diff --git a/python/pops/interfaces.py b/python/pops/interfaces.py index fdb7533f2..4662da304 100644 --- a/python/pops/interfaces.py +++ b/python/pops/interfaces.py @@ -170,6 +170,7 @@ def resolve(name: str) -> ComponentInterface: Tagger = resolve("tagger") Clustering = resolve("clustering") Transfer = resolve("transfer") +Reflux = resolve("reflux") FieldSolver = resolve("field_solver") Writer = resolve("writer") FieldTopology = resolve("field_topology") @@ -177,6 +178,6 @@ def resolve(name: str) -> ComponentInterface: __all__ = [ "ComponentInterface", "resolve", "NumericalFlux", "GhostBoundary", - "FieldBoundaryClosure", "Tagger", "Clustering", "Transfer", + "FieldBoundaryClosure", "Tagger", "Clustering", "Transfer", "Reflux", "FieldSolver", "Writer", "FieldTopology", ] diff --git a/python/pops/model/_generated_component_schema.py b/python/pops/model/_generated_component_schema.py index 84612fb23..7c829113c 100644 --- a/python/pops/model/_generated_component_schema.py +++ b/python/pops/model/_generated_component_schema.py @@ -3,8 +3,8 @@ COMPONENT_CATALOG_SCHEMA_VERSION = 1 COMPONENT_MANIFEST_SCHEMA_VERSION = 2 -COMPONENT_CATALOG_SHA256 = '84c68fcee96663f71e7e7fa7589ec1ddee0d1037e741a678e4afd83c9749620a' -COMPONENT_CATALOG_SEMANTIC_SHA256 = 'c0e14d4a3dd082612d052b2ce293e17de712c6916e39f430d0ff311ed0f24ef8' +COMPONENT_CATALOG_SHA256 = '5c991781aadafd3165dccb4642086c8b20fbd4a83ee8e462f1e29078ecb0d1c4' +COMPONENT_CATALOG_SEMANTIC_SHA256 = 'b29e5d1d811234bd55bfce83ffdf810ed95868392d1ac1da5644f8886620e129' COMPONENT_INTERFACE_SPECS = ({'name': 'requirement', 'method': 'requirements', 'required_args': 0}, {'name': 'lowering', 'method': 'lower', 'required_args': 1}, {'name': 'stencil', 'method': 'stencil', 'required_args': 0}, diff --git a/python/pops/runtime/_generated_component_routes.py b/python/pops/runtime/_generated_component_routes.py index 41b38a9e8..787947838 100644 --- a/python/pops/runtime/_generated_component_routes.py +++ b/python/pops/runtime/_generated_component_routes.py @@ -9,11 +9,11 @@ CAPABILITY_VOCAB_VERSION = 2 -COMPONENT_CATALOG_SHA256 = '84c68fcee96663f71e7e7fa7589ec1ddee0d1037e741a678e4afd83c9749620a' +COMPONENT_CATALOG_SHA256 = '5c991781aadafd3165dccb4642086c8b20fbd4a83ee8e462f1e29078ecb0d1c4' -COMPONENT_CATALOG_SEMANTIC_SHA256 = 'c0e14d4a3dd082612d052b2ce293e17de712c6916e39f430d0ff311ed0f24ef8' +COMPONENT_CATALOG_SEMANTIC_SHA256 = 'b29e5d1d811234bd55bfce83ffdf810ed95868392d1ac1da5644f8886620e129' -ROUTE_REGISTRY_SIGNATURE = 'v2:c0e14d4a3dd082612d052b2ce293e17de712c6916e39f430d0ff311ed0f24ef8' +ROUTE_REGISTRY_SIGNATURE = 'v2:b29e5d1d811234bd55bfce83ffdf810ed95868392d1ac1da5644f8886620e129' ROUTE_TABLES = {'riemann': (('rusanov', 'pops::RusanovFlux', diff --git a/schemas/component_catalog.v2.json b/schemas/component_catalog.v2.json index c48a03d1b..178196ea7 100644 --- a/schemas/component_catalog.v2.json +++ b/schemas/component_catalog.v2.json @@ -158,6 +158,16 @@ "facets": ["stencil", "lowering"], "operations": ["apply"] }, + { + "id": 6, + "name": "reflux", + "uri": "pops://interfaces/reflux", + "version": 1, + "cpp_table": "PopsRefluxApiV1", + "hot_path": true, + "facets": ["stencil", "lowering", "effects"], + "operations": ["apply_interface_batch"] + }, { "id": 7, "name": "field_solver", diff --git a/scripts/generate_component_catalog.py b/scripts/generate_component_catalog.py index 655377024..8a6a0cb75 100644 --- a/scripts/generate_component_catalog.py +++ b/scripts/generate_component_catalog.py @@ -649,6 +649,22 @@ def _render_component_abi(catalog: dict[str, Any], digest: str) -> str: % (row["name"].upper(), row["version"], row["cpp_table"]) for row in catalog["native_interface_abis"] ) + table_complete_rows = "\n".join( + """ case POPS_NATIVE_INTERFACE_%s_V%d: { + if (table_size < sizeof(%s)) return false; + const auto* api = static_cast(table); + return %s; + }""" + % ( + row["name"].upper(), + row["version"], + row["cpp_table"], + row["cpp_table"], + " && ".join("api->%s != nullptr" % operation + for operation in row["operations"]), + ) + for row in catalog["native_interface_abis"] + ) return f'''#pragma once // Generated by scripts/generate_component_catalog.py; DO NOT EDIT. @@ -1081,6 +1097,41 @@ def _render_component_abi(catalog: dict[str, Any], digest: str) -> str: PopsTransferApplyFnV1 apply; }} PopsTransferApiV1; +// Reflux providers are patch-local numerical kernels only. PoPS retains sole ownership of the +// time-integrated flux ledger, interface topology, MPI reduction, transaction and state update. +// Each face contains coarse/fine fluxes already integrated in time and averaged onto the same +// coarse face. The provider writes, but never applies, side*(fine-coarse)/dx into `correction`. +typedef enum PopsRefluxFaceSideV1 {{ + POPS_REFLUX_FACE_LOW_V1 = -1, + POPS_REFLUX_FACE_HIGH_V1 = 1 +}} PopsRefluxFaceSideV1; +typedef struct PopsRefluxFaceV1 {{ + uint32_t struct_size; + const char* interface_identity; + int32_t axis; + PopsRefluxFaceSideV1 side; + double inverse_coarse_cell_spacing; + PopsConstFieldViewV1 coarse_integrated_flux; + PopsConstFieldViewV1 fine_integrated_flux; + PopsFieldViewV1 correction; +}} PopsRefluxFaceV1; +typedef struct PopsRefluxRequestV1 {{ + uint32_t struct_size; + const char* transition_identity; + int32_t parent_level; + int32_t child_level; + size_t face_count; + const PopsRefluxFaceV1* faces; + PopsLogicalTimeV1 logical_time; + PopsExecutionContextV1 execution; +}} PopsRefluxRequestV1; +typedef int32_t (*PopsRefluxApplyInterfaceBatchFnV1)( + void*, const PopsRefluxRequestV1*, PopsComponentStatusV1*); +typedef struct PopsRefluxApiV1 {{ + PopsComponentTableHeaderV1 header; + PopsRefluxApplyInterfaceBatchFnV1 apply_interface_batch; +}} PopsRefluxApiV1; + typedef struct PopsFieldPatchMetadataV1 {{ uint32_t struct_size; size_t global_patch_index; @@ -1364,6 +1415,15 @@ def _render_component_abi(catalog: dict[str, Any], digest: str) -> str: }} return nullptr; }} +inline bool generated_native_interface_table_is_complete( + PopsNativeInterfaceIdV1 id, const void* table, size_t table_size) noexcept {{ + if (table == nullptr) + return false; + switch (id) {{ +{table_complete_rows} + }} + return false; +}} }} // namespace pops::component #endif // clang-format on From bbe30723d9a44ffa5001d4b82e7f81e6c0e04a44 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:02:45 +0200 Subject: [PATCH 002/109] feat(runtime): validate and invoke local Reflux kernels --- .../runtime/dynamic/component_consumers.hpp | 75 +++++++++++++++++ .../pops/runtime/dynamic/component_loader.hpp | 3 + .../runtime/test_component_interfaces.cpp | 81 +++++++++++++++++++ .../test_external_interface_backend.py | 31 ++++++- 4 files changed, 189 insertions(+), 1 deletion(-) diff --git a/include/pops/runtime/dynamic/component_consumers.hpp b/include/pops/runtime/dynamic/component_consumers.hpp index ccdaeb506..586eae878 100644 --- a/include/pops/runtime/dynamic/component_consumers.hpp +++ b/include/pops/runtime/dynamic/component_consumers.hpp @@ -596,6 +596,81 @@ inline int apply_transfer(const PopsTransferApiV1& api, void* state, return api.apply(state, &request, &status); } +template +inline bool same_reflux_face_shape(const Left& left, const Right& right) { + if (left.dimension != right.dimension || left.component_count != right.component_count || + left.scalar_type != right.scalar_type || left.memory_space != right.memory_space) + return false; + for (std::int32_t axis = 0; axis < 3; ++axis) + if (left.extents[axis] != right.extents[axis] || + left.ghost_lower[axis] != right.ghost_lower[axis] || + left.ghost_upper[axis] != right.ghost_upper[axis]) + return false; + return true; +} + +inline int apply_reflux_interface_batch(const PopsRefluxApiV1& api, void* state, + const PopsRefluxRequestV1& request, + PopsComponentStatusV1& status) { + require_operation(api.apply_interface_batch != nullptr, "apply_interface_batch"); + if (request.struct_size < sizeof(PopsRefluxRequestV1) || + !component_text(request.transition_identity) || request.parent_level < 0 || + request.child_level != request.parent_level + 1 || request.face_count == 0 || + request.faces == nullptr || request.logical_time.level != request.parent_level) + throw std::invalid_argument("reflux request is incomplete"); + validate_logical_time(request.logical_time); + validate_noncollective_execution_context(request.execution); + + std::unordered_set identities; + for (std::size_t index = 0; index < request.face_count; ++index) { + const auto& face = request.faces[index]; + if (face.struct_size < sizeof(PopsRefluxFaceV1) || !component_text(face.interface_identity) || + !identities.insert(face.interface_identity).second || face.axis < 0 || face.axis >= 2 || + (face.side != POPS_REFLUX_FACE_LOW_V1 && face.side != POPS_REFLUX_FACE_HIGH_V1) || + !std::isfinite(face.inverse_coarse_cell_spacing) || face.inverse_coarse_cell_spacing <= 0.0) + throw std::invalid_argument("reflux face descriptor is incomplete"); + + validate_execution_field(request.execution, face.coarse_integrated_flux, + "reflux coarse integrated flux"); + validate_execution_field(request.execution, face.fine_integrated_flux, + "reflux fine integrated flux"); + validate_execution_field(request.execution, face.correction, "reflux correction"); + const auto centering_axis = 1u << static_cast(face.axis); + if (face.coarse_integrated_flux.centering != POPS_FIELD_CENTERING_FACE_V1 || + face.fine_integrated_flux.centering != POPS_FIELD_CENTERING_FACE_V1 || + face.coarse_integrated_flux.centering_axes != centering_axis || + face.fine_integrated_flux.centering_axes != centering_axis || + face.correction.centering != POPS_FIELD_CENTERING_CELL_V1 || + face.correction.centering_axes != 0 || + face.coarse_integrated_flux.ownership != POPS_FIELD_OWNERSHIP_RUNTIME_BORROWED_V1 || + face.fine_integrated_flux.ownership != POPS_FIELD_OWNERSHIP_RUNTIME_BORROWED_V1 || + face.correction.ownership != POPS_FIELD_OWNERSHIP_RUNTIME_BORROWED_V1 || + !same_reflux_face_shape(face.coarse_integrated_flux, face.fine_integrated_flux) || + !same_reflux_face_shape(face.coarse_integrated_flux, face.correction) || + face.coarse_integrated_flux.extents[face.axis] != 1 || + std::string(face.coarse_integrated_flux.layout_identity) != + face.correction.layout_identity || + std::string(face.coarse_integrated_flux.patch_identity) != face.correction.patch_identity) + throw std::invalid_argument( + "reflux face fluxes and correction disagree on shape, centering or ownership"); + for (std::int32_t axis = 0; axis < face.coarse_integrated_flux.dimension; ++axis) + if (face.coarse_integrated_flux.ghost_lower[axis] != 0 || + face.coarse_integrated_flux.ghost_upper[axis] != 0) + throw std::invalid_argument("reflux face views cannot carry ghost cells"); + } + + status = unwritten_component_status(); + const int code = api.apply_interface_batch(state, &request, &status); + if (!component_status_is_well_formed(status)) + throw std::runtime_error("native Reflux component returned an invalid status"); + if ((code == 0) != (status.code == 0) || + (code == 0 && status.action != POPS_COMPONENT_CONTINUE_V1) || + (code != 0 && status.action == POPS_COMPONENT_CONTINUE_V1) || + (code != 0 && !component_text(status.reason))) + throw std::runtime_error("native Reflux component returned an inconsistent outcome"); + return code; +} + inline std::string writer_geometry_key(const char* layout, std::int32_t level) { return std::string(layout) + "\n" + std::to_string(level); } diff --git a/include/pops/runtime/dynamic/component_loader.hpp b/include/pops/runtime/dynamic/component_loader.hpp index 0cb9ffc46..ce9483ae0 100644 --- a/include/pops/runtime/dynamic/component_loader.hpp +++ b/include/pops/runtime/dynamic/component_loader.hpp @@ -392,6 +392,9 @@ class LoadedComponent final { if ((header->prepare == nullptr) != (header->destroy == nullptr)) throw std::runtime_error( "native component interface prepare/destroy callbacks must be paired"); + if (!generated_native_interface_table_is_complete(row.interface_id, row.table, + row.table_size)) + throw std::runtime_error("native component interface table misses a required operation"); } for (const auto& required : expected.interfaces) { bool found = false; diff --git a/tests/cpp/unit/runtime/test_component_interfaces.cpp b/tests/cpp/unit/runtime/test_component_interfaces.cpp index 717c42601..57d4cc2ff 100644 --- a/tests/cpp/unit/runtime/test_component_interfaces.cpp +++ b/tests/cpp/unit/runtime/test_component_interfaces.cpp @@ -53,6 +53,12 @@ struct TransferComponent { std::string restart() const { return "stateless"; } }; +struct RefluxComponent { + int stencil() const { return 1; } + std::string lower(Context&) const { return "integrated-interface-correction"; } + std::vector effects() const { return {"local-correction"}; } +}; + struct SolverComponent { pops::component::EvaluationOutcome evaluate(Context&) const { return pops::component::EvaluationOutcome::reject("non-converged"); @@ -89,6 +95,9 @@ static_assert(pops::component::Lowering); static_assert(pops::component::Effects); static_assert(pops::component::Stencil); static_assert(pops::component::Restart); +static_assert(pops::component::Stencil); +static_assert(pops::component::Lowering); +static_assert(pops::component::Effects); static_assert(pops::component::FallibleEvaluation); static_assert(pops::component::Restart); static_assert(pops::component::Format); @@ -731,6 +740,78 @@ TEST(ComponentInterfaces, ExactAbiConsumersExecuteEveryClosedScientificFamily) { EXPECT_THROW(pops::component::apply_transfer(transfer_api, nullptr, wrong_transfer_shape, status), std::invalid_argument); + std::array coarse_integrated_flux{1.0, 2.0}; + std::array fine_integrated_flux{3.0, 6.0}; + std::array reflux_correction{}; + PopsRefluxApiV1 reflux_api{ + abi_header(sizeof(PopsRefluxApiV1), POPS_NATIVE_INTERFACE_REFLUX_V1), + +[](void*, const PopsRefluxRequestV1* request, PopsComponentStatusV1* result) { + for (std::size_t face_index = 0; face_index < request->face_count; ++face_index) { + const auto& face = request->faces[face_index]; + const auto* coarse = static_cast(face.coarse_integrated_flux.data); + const auto* fine = static_cast(face.fine_integrated_flux.data); + auto* correction = static_cast(face.correction.data); + const std::size_t points = + pops::component::field_point_count(face.coarse_integrated_flux); + for (std::size_t point = 0; point < points; ++point) + correction[point] = static_cast(face.side) * (fine[point] - coarse[point]) * + face.inverse_coarse_cell_spacing; + } + *result = ok_status(); + return 0; + }}; + auto coarse_face = abi::const_field_view(coarse_integrated_flux.data(), 1, 2, 1, "parent::layout", + "parent::patch"); + coarse_face.centering = POPS_FIELD_CENTERING_FACE_V1; + coarse_face.centering_axes = 1u; + auto fine_face = + abi::const_field_view(fine_integrated_flux.data(), 1, 2, 1, "child::layout", "child::patch"); + fine_face.centering = POPS_FIELD_CENTERING_FACE_V1; + fine_face.centering_axes = 1u; + PopsRefluxFaceV1 reflux_face{ + sizeof(PopsRefluxFaceV1), + "transition::0-to-1/x-low", + 0, + POPS_REFLUX_FACE_LOW_V1, + 2.0, + coarse_face, + fine_face, + abi::field_view(reflux_correction.data(), 1, 2, 1, "parent::layout", "parent::patch")}; + PopsRefluxRequestV1 reflux_request{sizeof(PopsRefluxRequestV1), + "transition::0-to-1", + 0, + 1, + 1, + &reflux_face, + abi::logical_time(), + abi::noncollective_host_execution_context()}; + EXPECT_TRUE(pops::component::generated_native_interface_table_is_complete( + POPS_NATIVE_INTERFACE_REFLUX_V1, &reflux_api, sizeof(reflux_api))); + EXPECT_EQ( + pops::component::apply_reflux_interface_batch(reflux_api, nullptr, reflux_request, status), + 0); + EXPECT_EQ(reflux_correction, (std::array{-4.0, -8.0})); + + auto incomplete_reflux_api = reflux_api; + incomplete_reflux_api.apply_interface_batch = nullptr; + EXPECT_FALSE(pops::component::generated_native_interface_table_is_complete( + POPS_NATIVE_INTERFACE_REFLUX_V1, &incomplete_reflux_api, sizeof(incomplete_reflux_api))); + EXPECT_THROW(pops::component::apply_reflux_interface_batch(incomplete_reflux_api, nullptr, + reflux_request, status), + std::runtime_error); + auto collective_reflux = reflux_request; + collective_reflux.execution = execution; + EXPECT_THROW( + pops::component::apply_reflux_interface_batch(reflux_api, nullptr, collective_reflux, status), + std::invalid_argument); + auto malformed_reflux = reflux_request; + auto malformed_face = reflux_face; + malformed_face.correction.layout_identity = "other::parent-layout"; + malformed_reflux.faces = &malformed_face; + EXPECT_THROW( + pops::component::apply_reflux_interface_batch(reflux_api, nullptr, malformed_reflux, status), + std::invalid_argument); + auto overflowing_ghosts = abi::const_field_view(tag_values.data(), 2, 2); overflowing_ghosts.ghost_lower[0] = std::numeric_limits::max(); overflowing_ghosts.ghost_upper[0] = 1; diff --git a/tests/python/integration/native_loader/test_external_interface_backend.py b/tests/python/integration/native_loader/test_external_interface_backend.py index 07617eaa9..48e7b6364 100644 --- a/tests/python/integration/native_loader/test_external_interface_backend.py +++ b/tests/python/integration/native_loader/test_external_interface_backend.py @@ -5,23 +5,52 @@ import json from pathlib import Path +import pytest + from pops import interfaces from pops import _generated_component_interfaces as generated +from pops.model import ComponentManifest def test_all_required_native_families_are_generated_data_only_contracts(): expected = { "numerical_flux", "ghost_boundary", "field_boundary_closure", "tagger", - "clustering", "transfer", "field_solver", "writer", "field_topology", + "clustering", "transfer", "reflux", "field_solver", "writer", "field_topology", } resolved = {name: interfaces.resolve(name) for name in expected} assert set(resolved) == expected assert len({value.abi_id for value in resolved.values()}) == len(expected) + assert sorted(value.abi_id for value in resolved.values()) == list(range(10)) assert all(value.table_symbol == "pops_component_interface_v1" for value in resolved.values()) assert all(value.operations for value in resolved.values()) +def test_reflux_is_exact_generated_id_6_and_incomplete_conformer_is_refused(): + interface = interfaces.Reflux + assert interface.abi_id == 6 + assert interface.uri == "pops://interfaces/reflux" + assert interface.cpp_table == "PopsRefluxApiV1" + assert interface.operations == ("apply_interface_batch",) + + incomplete_signature = interface.signature_declaration() + incomplete_signature["operations"] = () + manifest = ComponentManifest( + uri="pops://external.test/reflux/incomplete", + component_type="reflux", + version="1.0.0", + facets=interface.facets, + signature={"native_interface": incomplete_signature}, + interfaces=interface.manifest_declarations(), + target={"variants": [{ + "dimension": 2, "scalar": "float64", "device": "cpu", "features": [], + }]}, + entry_points={"interface_table": "pops_component_interface_v1"}, + ) + with pytest.raises(ValueError, match="does not carry the generated native interface identity"): + interface.require_manifest(manifest) + + def test_python_native_component_boundary_has_no_ffi_or_test_owned_backend(): root = Path(__file__).resolve().parents[4] production = ( From c418145c3a5d98730abb513918aa72c0991dd1fd Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:05:51 +0200 Subject: [PATCH 003/109] docs(components): bound external Reflux authority --- .../SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md index f7652d26d..d279bfee2 100644 --- a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md +++ b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md @@ -1422,7 +1422,9 @@ Le contrat `Reflux` v1 est volontairement livré avant son branchement dans `PreparedAmrProgramRefluxTransition` : catalogue, manifest, loader et consumer typé peuvent qualifier un conformer, mais le runtime AMR continue d'utiliser son kernel interne tant qu'un adaptateur préparé ne peut pas fournir les vues locales sans dupliquer le ledger ni transférer l'autorité collective. Une -configuration AMR ne prétend donc pas encore avoir sélectionné un provider `Reflux` externe. +configuration AMR ne prétend donc pas encore avoir sélectionné un provider `Reflux` externe. Cette +première qualification est limitée à la cible 2D, `float64`, CPU déjà admise par le loader de +composants ; elle ne constitue pas une promesse GPU. Les champs sémantiques inconnus, capacités sans preuve, collisions d'identité et entry points manquants sont refusés. Un vieux manifest n'est pas « réparé » silencieusement. From 821b7c66ef5e879aa99c238e9b5862213025bd97 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:06:40 +0200 Subject: [PATCH 004/109] test(components): pin native interface id parity --- .../test_external_interface_backend.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/python/integration/native_loader/test_external_interface_backend.py b/tests/python/integration/native_loader/test_external_interface_backend.py index 48e7b6364..6d510c6a8 100644 --- a/tests/python/integration/native_loader/test_external_interface_backend.py +++ b/tests/python/integration/native_loader/test_external_interface_backend.py @@ -20,7 +20,18 @@ def test_all_required_native_families_are_generated_data_only_contracts(): resolved = {name: interfaces.resolve(name) for name in expected} assert set(resolved) == expected assert len({value.abi_id for value in resolved.values()}) == len(expected) - assert sorted(value.abi_id for value in resolved.values()) == list(range(10)) + assert {name: value.abi_id for name, value in resolved.items()} == { + "numerical_flux": 0, + "ghost_boundary": 1, + "field_boundary_closure": 2, + "tagger": 3, + "clustering": 4, + "transfer": 5, + "reflux": 6, + "field_solver": 7, + "writer": 8, + "field_topology": 9, + } assert all(value.table_symbol == "pops_component_interface_v1" for value in resolved.values()) assert all(value.operations for value in resolved.values()) From 63130f244d459ee507f2b3ab00d7e8c0d77efe06 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:18:50 +0200 Subject: [PATCH 005/109] test(mpi): exercise interface scheduler on execution lane --- ...est_mpi_multiblock_interface_scheduler.cpp | 39 +++++++++++++++++-- ...multiblock_interface_communicator_fence.py | 33 ++++++++++++++++ 2 files changed, 68 insertions(+), 4 deletions(-) create mode 100644 tests/python/architecture/test_multiblock_interface_communicator_fence.py diff --git a/tests/cpp/integration/mpi/test_mpi_multiblock_interface_scheduler.cpp b/tests/cpp/integration/mpi/test_mpi_multiblock_interface_scheduler.cpp index 70a979e35..778759681 100644 --- a/tests/cpp/integration/mpi/test_mpi_multiblock_interface_scheduler.cpp +++ b/tests/cpp/integration/mpi/test_mpi_multiblock_interface_scheduler.cpp @@ -16,7 +16,33 @@ using namespace pops::runtime::multiblock; namespace { -PopsExecutionContextV1 mpi_world_execution() { +class ScopedMpiCommunicator { + public: + explicit ScopedMpiCommunicator(MPI_Comm source) { + if (MPI_Comm_dup(source, &communicator_) != MPI_SUCCESS) + throw std::runtime_error("MPI_Comm_dup failed for the interface scheduler test lane"); + if (MPI_Comm_set_errhandler(communicator_, MPI_ERRORS_RETURN) != MPI_SUCCESS) { + MPI_Comm_free(&communicator_); + throw std::runtime_error( + "MPI_Comm_set_errhandler failed for the interface scheduler test lane"); + } + } + + ~ScopedMpiCommunicator() { + if (communicator_ != MPI_COMM_NULL) + MPI_Comm_free(&communicator_); + } + + ScopedMpiCommunicator(const ScopedMpiCommunicator&) = delete; + ScopedMpiCommunicator& operator=(const ScopedMpiCommunicator&) = delete; + + MPI_Comm get() const { return communicator_; } + + private: + MPI_Comm communicator_ = MPI_COMM_NULL; +}; + +PopsExecutionContextV1 mpi_lane_execution(MPI_Comm communicator) { return {sizeof(PopsExecutionContextV1), 1u, "test::mpi-multiblock-execution", @@ -30,9 +56,9 @@ PopsExecutionContextV1 mpi_world_execution() { POPS_PRECISION_FLOAT64_V1, 0, "host::synchronous", - static_cast(MPI_Comm_c2f(MPI_COMM_WORLD)), + static_cast(MPI_Comm_c2f(communicator)), static_cast(MPI_Type_c2f(MPI_DOUBLE)), - "MPI_COMM_WORLD", + "test::mpi-multiblock-interface-lane", "MPI_DOUBLE"}; } @@ -95,6 +121,11 @@ int run_mpi_multiblock_interface_scheduler(int argc, char** argv) { { try { require(n_ranks() == 2); + const ScopedMpiCommunicator interface_lane(MPI_COMM_WORLD); + int world_relation = MPI_UNEQUAL; + require(MPI_Comm_compare(interface_lane.get(), MPI_COMM_WORLD, &world_relation) == + MPI_SUCCESS); + require(world_relation == MPI_CONGRUENT); const Box2D left_domain{{0, 0}, {1, 3}}; const Box2D right_domain{{2, 0}, {3, 3}}; @@ -124,7 +155,7 @@ int run_mpi_multiblock_interface_scheduler(int argc, char** argv) { const Geometry left_geometry{left_domain, Real(0), Real(1), Real(0), Real(1)}; const Geometry right_geometry{right_domain, Real(1), Real(2), Real(0), Real(1)}; - const PopsExecutionContextV1 execution = mpi_world_execution(); + const PopsExecutionContextV1 execution = mpi_lane_execution(interface_lane.get()); const BoundaryEvaluationPoint point{"clock.mpi-interface", 3, 0, 0, 1, amr::Rational(1, 1), 0.125, 0.375}; diff --git a/tests/python/architecture/test_multiblock_interface_communicator_fence.py b/tests/python/architecture/test_multiblock_interface_communicator_fence.py new file mode 100644 index 000000000..d472965ef --- /dev/null +++ b/tests/python/architecture/test_multiblock_interface_communicator_fence.py @@ -0,0 +1,33 @@ +"""ADC-683 fences for execution-lane-owned multi-block interface collectives.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCHEDULER = ROOT / "include/pops/runtime/multiblock/interface_flux_scheduler.hpp" + + +def _function(source: str, signature: str) -> str: + start = source.index(signature) + opening_brace = source.index("{", start) + depth = 0 + for offset in range(opening_brace, len(source)): + token = source[offset] + if token == "{": + depth += 1 + elif token == "}": + depth -= 1 + if depth == 0: + return source[start : offset + 1] + raise AssertionError(f"unterminated C++ function {signature}") + + +def test_interface_scheduler_hot_path_never_falls_back_to_mpi_world(): + source = SCHEDULER.read_text(encoding="utf-8") + consensus = _function(source, "static void require_distributed_flux_consensus_(") + apply_one = _function(source, "static void apply_one_(") + + assert "MPI_COMM_WORLD" not in consensus + assert "MPI_COMM_WORLD" not in apply_one + assert "const CommunicatorView& communicator" in consensus + assert "prepared.communicator" in apply_one From 2e1163945a224b2112b2b338edb7e210c6374e40 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:22:10 +0200 Subject: [PATCH 006/109] feat(mpi): execute interface collectives on resolved communicator --- .../multiblock/interface_flux_scheduler.hpp | 181 ++++++++++++------ 1 file changed, 122 insertions(+), 59 deletions(-) diff --git a/include/pops/runtime/multiblock/interface_flux_scheduler.hpp b/include/pops/runtime/multiblock/interface_flux_scheduler.hpp index 80325ad91..a240ab129 100644 --- a/include/pops/runtime/multiblock/interface_flux_scheduler.hpp +++ b/include/pops/runtime/multiblock/interface_flux_scheduler.hpp @@ -114,8 +114,13 @@ class InterfaceFluxScheduler { const PopsExecutionContextV1& execution, InterfaceFluxEvaluatorFactory evaluator_factory) { const bool collective_world = comm_active() && n_ranks() > 1; + const CommunicatorView admission_communicator = + collective_world ? world_communicator_view() : CommunicatorView{}; bool distributed = false; + CommunicatorView execution_communicator; + int communicator_rank = 0; int communicator_size = 1; + std::string communicator_identity = "serial"; int component_count = 0; int left_faces = 0; Real left_normal = Real(0); @@ -135,34 +140,63 @@ class InterfaceFluxScheduler { if (route.left_side == route.right_side) throw std::invalid_argument("multi-block interface faces do not have opposite orientation"); component::validate_execution_context(execution); - const std::string communicator_identity(execution.communicator_identity); - if (communicator_identity == "MPI_COMM_WORLD") { + communicator_identity.assign(execution.communicator_identity); + if (communicator_identity != "serial" && + communicator_identity != POPS_EXECUTION_NONCOLLECTIVE_IDENTITY_V1) { #ifdef POPS_HAS_MPI if (!comm_active()) throw std::invalid_argument( - "multi-block interface MPI_COMM_WORLD capability is not active"); - int communicator_relation = MPI_UNEQUAL; - ::pops::detail::require_mpi_success( - MPI_Comm_compare(MPI_Comm_f2c(static_cast(execution.communicator_f_handle)), - MPI_COMM_WORLD, &communicator_relation), - "MPI_Comm_compare(interface execution context)"); - if (communicator_relation != MPI_IDENT || + "multi-block interface communicator capability is not active"); + const MPI_Comm communicator = + MPI_Comm_f2c(static_cast(execution.communicator_f_handle)); + if (communicator == MPI_COMM_NULL || MPI_Type_f2c(static_cast(execution.communicator_datatype_f_handle)) != MPI_DOUBLE) throw std::invalid_argument( - "multi-block interface execution handles do not identify exact " - "MPI_COMM_WORLD/MPI_DOUBLE"); - communicator_size = n_ranks(); + "multi-block interface execution handles do not identify a live " + "communicator/MPI_DOUBLE authority"); + int communicator_relation = MPI_UNEQUAL; + ::pops::detail::require_mpi_success( + MPI_Comm_compare(communicator, MPI_COMM_WORLD, &communicator_relation), + "MPI_Comm_compare(interface field rank space)"); + if (communicator_relation != MPI_IDENT && communicator_relation != MPI_CONGRUENT) + throw std::invalid_argument( + "multi-block interface communicator must preserve the field rank space"); + execution_communicator = CommunicatorView{communicator}; + communicator_rank = execution_communicator.rank(); + communicator_size = execution_communicator.size(); distributed = communicator_size > 1; #else throw std::invalid_argument( - "multi-block interface scheduler received MPI_COMM_WORLD from a serial build"); + "multi-block interface scheduler received a distributed context from a serial build"); #endif + } else if (communicator_identity == POPS_EXECUTION_NONCOLLECTIVE_IDENTITY_V1) { + throw std::invalid_argument( + "multi-block interface scheduler requires collective execution authority"); #ifdef POPS_HAS_MPI } else if (comm_active() && n_ranks() > 1) { throw std::invalid_argument( "multi-block interface cannot use a serial execution identity in an active " "multi-rank MPI world"); +#endif + } + if (!interfaces_.empty()) { + const PreparedInterface& existing = interfaces_.front(); + if (existing.communicator_identity != communicator_identity || + existing.communicator_size != communicator_size) + throw std::invalid_argument( + "multi-block interface routes require one exact execution communicator"); +#ifdef POPS_HAS_MPI + if (distributed) { + int relation = MPI_UNEQUAL; + ::pops::detail::require_mpi_success( + MPI_Comm_compare(existing.communicator.native_handle(), + execution_communicator.native_handle(), &relation), + "MPI_Comm_compare(installed interface communicators)"); + if (relation != MPI_IDENT) + throw std::invalid_argument( + "multi-block interface routes require the same communicator context"); + } #endif } if (left_state.box_array().size() < 1 || right_state.box_array().size() < 1) @@ -256,21 +290,24 @@ class InterfaceFluxScheduler { throw std::invalid_argument( "multi-block interface faces do not coincide in physical space"); - left_cells = boundary_cells_(left_state, route.left_axis, route.left_side, left_faces); - right_cells = boundary_cells_(right_state, route.right_axis, route.right_side, right_faces); + left_cells = boundary_cells_(left_state, route.left_axis, route.left_side, left_faces, + communicator_rank); + right_cells = boundary_cells_(right_state, route.right_axis, route.right_side, right_faces, + communicator_rank); } catch (...) { structural_failure = std::current_exception(); } - finish_collective_preflight_(collective_world, structural_failure, + finish_collective_preflight_(admission_communicator, structural_failure, "route/layout/execution preflight"); - if (distributed && !registry_agrees_across_ranks_()) + if (distributed && !registry_agrees_across_ranks_(execution_communicator)) throw std::runtime_error("multi-block interface prepared registry differs across MPI ranks"); const std::string collective_identity = collective_plan_identity_( route, left_state, left_geometry, right_state, right_geometry, left_normal, right_normal, - left_faces, component_count, communicator_size); + left_faces, component_count, communicator_identity, communicator_size); if (distributed && !all_ranks_agree_exact_ordered_byte_pairs( - {{std::string_view(route.identity), std::string_view(collective_identity)}})) + {{std::string_view(route.identity), std::string_view(collective_identity)}}, + execution_communicator)) throw std::runtime_error( "multi-block interface prepared route/layout differs across MPI ranks"); PreparedInterface prepared; @@ -290,7 +327,10 @@ class InterfaceFluxScheduler { left_faces, component_count, distributed, + execution_communicator, + communicator_rank, communicator_size, + communicator_identity, collective_identity, InterfaceFluxEvaluator{}, 0}; @@ -305,7 +345,7 @@ class InterfaceFluxScheduler { } catch (...) { materialization_failure = std::current_exception(); } - finish_collective_preflight_(distributed, materialization_failure, + finish_collective_preflight_(execution_communicator, materialization_failure, "prepared-route materialization"); // Component prepare may allocate resources or have observable external effects. Invoke it only // after every route/layout/geometry capability has been proved, but before mutating the scheduler @@ -321,7 +361,8 @@ class InterfaceFluxScheduler { } catch (...) { evaluator_prepare_failure = std::current_exception(); } - finish_collective_preflight_(distributed, evaluator_prepare_failure, "evaluator preparation"); + finish_collective_preflight_(execution_communicator, evaluator_prepare_failure, + "evaluator preparation"); prepared.evaluator = std::move(evaluator); interfaces_.push_back(std::move(prepared)); } @@ -341,18 +382,26 @@ class InterfaceFluxScheduler { void apply(const BoundaryEvaluationPoint& point, const std::vector& states, const std::vector& rhs, InterfaceFluxFragmentPublication* publication = nullptr) { - const bool collective_world = comm_active() && n_ranks() > 1; + if (interfaces_.empty()) { + validate_point_(point); + if (publication != nullptr) + validate_fragment_publication_(point, *publication); + return; + } + const CommunicatorView execution_communicator = interfaces_.front().communicator; + const bool collective = execution_communicator.active() && execution_communicator.size() > 1; std::exception_ptr point_failure; try { validate_point_(point); } catch (...) { point_failure = std::current_exception(); } - finish_collective_preflight_(collective_world, point_failure, "evaluation-point preflight"); + finish_collective_preflight_(execution_communicator, point_failure, + "evaluation-point preflight"); std::exception_ptr publication_failure; try { if (publication != nullptr) { - if (collective_world) + if (collective) throw std::runtime_error( "AMR interface-flux fragment publication does not yet support distributed MPI"); validate_fragment_publication_(point, *publication); @@ -360,13 +409,14 @@ class InterfaceFluxScheduler { } catch (...) { publication_failure = std::current_exception(); } - finish_collective_preflight_(collective_world, publication_failure, + finish_collective_preflight_(execution_communicator, publication_failure, "interface-fragment publication preflight"); - if (collective_world && !registry_agrees_across_ranks_()) + if (collective && !registry_agrees_across_ranks_(execution_communicator)) throw std::runtime_error("multi-block interface prepared registry differs across MPI ranks"); const std::string point_identity = collective_point_identity_(point); - if (collective_world && !all_ranks_agree_exact_ordered_byte_pairs( - {{std::string_view("point"), std::string_view(point_identity)}})) + if (collective && !all_ranks_agree_exact_ordered_byte_pairs( + {{std::string_view("point"), std::string_view(point_identity)}}, + execution_communicator)) throw std::runtime_error( "multi-block interface BoundaryEvaluationPoint differs across MPI ranks"); @@ -398,11 +448,11 @@ class InterfaceFluxScheduler { } catch (...) { active_mask_failure = std::current_exception(); } - finish_collective_preflight_(prepared.distributed, active_mask_failure, + finish_collective_preflight_(prepared.communicator, active_mask_failure, "active-mask preflight"); if (prepared.distributed) { - const long minimum_active = all_reduce_min(active ? 1L : 0L); - const long maximum_active = all_reduce_max(active ? 1L : 0L); + const long minimum_active = all_reduce_min(active ? 1L : 0L, prepared.communicator); + const long maximum_active = all_reduce_max(active ? 1L : 0L, prepared.communicator); if (minimum_active != maximum_active) throw std::runtime_error("multi-block interface active mask differs across MPI ranks"); } @@ -487,7 +537,10 @@ class InterfaceFluxScheduler { int face_count = 0; int component_count = 0; bool distributed = false; + CommunicatorView communicator; + int communicator_rank = 0; int communicator_size = 1; + std::string communicator_identity; std::string collective_identity; InterfaceFluxEvaluator evaluator; std::size_t evaluation_count = 0; @@ -499,10 +552,12 @@ class InterfaceFluxScheduler { }; static_assert(std::is_nothrow_move_constructible_v); - static void finish_collective_preflight_(bool collective, const std::exception_ptr& local_failure, + static void finish_collective_preflight_(const CommunicatorView& communicator, + const std::exception_ptr& local_failure, const char* phase) { - const long failure_count = - collective ? all_reduce_sum(local_failure ? 1L : 0L) : (local_failure ? 1L : 0L); + const bool collective = communicator.active() && communicator.size() > 1; + const long failure_count = collective ? all_reduce_sum(local_failure ? 1L : 0L, communicator) + : (local_failure ? 1L : 0L); if (failure_count == 0) return; if (local_failure) @@ -552,9 +607,10 @@ class InterfaceFluxScheduler { static std::string collective_plan_identity_( const AxisAlignedInterface& route, const MultiFab& left_state, const Geometry& left_geometry, const MultiFab& right_state, const Geometry& right_geometry, Real left_normal, - Real right_normal, int face_count, int component_count, int communicator_size) { + Real right_normal, int face_count, int component_count, + std::string_view communicator_identity, int communicator_size) { std::string bytes; - append_identity_text_(bytes, "pops.multiblock.interface-plan.v1"); + append_identity_text_(bytes, "pops.multiblock.interface-plan.v2"); append_identity_text_(bytes, route.identity); append_identity_scalar_(bytes, static_cast(route.left_block)); append_identity_scalar_(bytes, static_cast(route.right_block)); @@ -580,6 +636,7 @@ class InterfaceFluxScheduler { append_identity_scalar_(bytes, right_normal); append_identity_scalar_(bytes, face_count); append_identity_scalar_(bytes, component_count); + append_identity_text_(bytes, communicator_identity); append_identity_scalar_(bytes, communicator_size); return bytes; } @@ -599,12 +656,12 @@ class InterfaceFluxScheduler { return bytes; } - bool registry_agrees_across_ranks_() const { + bool registry_agrees_across_ranks_(const CommunicatorView& communicator) const { std::vector> identities; identities.reserve(interfaces_.size()); for (const PreparedInterface& prepared : interfaces_) identities.emplace_back(prepared.route.identity, prepared.collective_identity); - return all_ranks_agree_exact_ordered_byte_pairs(identities); + return all_ranks_agree_exact_ordered_byte_pairs(identities, communicator); } static int tangential_count_(const Box2D& box, InterfaceAxis axis) { @@ -652,7 +709,8 @@ class InterfaceFluxScheduler { } static std::vector boundary_cells_(const MultiFab& field, InterfaceAxis axis, - InterfaceSide side, int face_count) { + InterfaceSide side, int face_count, + int communicator_rank) { const Box2D domain = field.box_array().bounding_box(); const int normal_axis = axis == InterfaceAxis::X ? 0 : 1; const int tangent_axis = 1 - normal_axis; @@ -676,7 +734,7 @@ class InterfaceFluxScheduler { throw std::invalid_argument( "multi-block interface boundary decomposition has a gap at one face cell"); const int local_owner = field.local_index_of(global_owner); - if ((field.dmap()[global_owner] == my_rank()) != (local_owner >= 0)) + if ((field.dmap()[global_owner] == communicator_rank) != (local_owner >= 0)) throw std::logic_error( "multi-block interface local ownership differs from its DistributionMapping"); cells.push_back(BoundaryCell{local_owner, i, j}); @@ -742,32 +800,34 @@ class InterfaceFluxScheduler { static bool runtime_field_matches_(const MultiFab& field, const std::vector& expected_boxes, - const std::vector& expected_ranks, int component_count) { + const std::vector& expected_ranks, int component_count, + int communicator_rank) { int expected_local_size = 0; for (const int owner : expected_ranks) - if (owner == my_rank()) + if (owner == communicator_rank) ++expected_local_size; return field.box_array().boxes() == expected_boxes && field.dmap().ranks() == expected_ranks && field.local_size() == expected_local_size && field.ncomp() == component_count; } static void require_distributed_flux_consensus_(std::vector& flux, - std::vector& reference) { + std::vector& reference, + const CommunicatorView& communicator) { #ifdef POPS_HAS_MPI if (reference.size() != flux.size()) throw std::logic_error("multi-block interface consensus scratch changed size"); std::copy(flux.begin(), flux.end(), reference.begin()); - ::pops::detail::require_mpi_success( - MPI_Bcast(reference.data(), static_cast(reference.size()), MPI_DOUBLE, 0, - MPI_COMM_WORLD), - "MPI_Bcast(multi-block shared flux)"); + broadcast_bytes_inplace(reinterpret_cast(reference.data()), + reference.size() * sizeof(Real), 0, communicator); const bool equal = std::memcmp(reference.data(), flux.data(), flux.size() * sizeof(Real)) == 0; - if (all_reduce_sum(equal ? 0L : 1L) != 0) + if (all_reduce_sum(equal ? 0L : 1L, communicator) != 0) throw std::runtime_error( "multi-block interface evaluator returned rank-dependent shared flux"); std::copy(reference.begin(), reference.end(), flux.begin()); #else (void)flux; + (void)reference; + (void)communicator; throw std::logic_error( "distributed multi-block flux consensus is unavailable in a serial build"); #endif @@ -776,19 +836,22 @@ class InterfaceFluxScheduler { static void apply_one_(PreparedInterface& prepared, const BoundaryEvaluationPoint& point, MultiFab& left_state, MultiFab& right_state, MultiFab& left_rhs, MultiFab& right_rhs, InterfaceFluxFragmentPublication* publication) { - if (prepared.distributed && (!comm_active() || n_ranks() != prepared.communicator_size)) - throw std::runtime_error("multi-block interface MPI world changed after route preparation"); + if (prepared.distributed && (!prepared.communicator.active() || + prepared.communicator.size() != prepared.communicator_size || + prepared.communicator.rank() != prepared.communicator_rank)) + throw std::runtime_error( + "multi-block interface execution communicator changed after route preparation"); const bool layouts_match = runtime_field_matches_(left_state, prepared.left_boxes, prepared.left_ranks, - prepared.component_count) && + prepared.component_count, prepared.communicator_rank) && runtime_field_matches_(right_state, prepared.right_boxes, prepared.right_ranks, - prepared.component_count) && + prepared.component_count, prepared.communicator_rank) && runtime_field_matches_(left_rhs, prepared.left_boxes, prepared.left_ranks, - prepared.component_count) && + prepared.component_count, prepared.communicator_rank) && runtime_field_matches_(right_rhs, prepared.right_boxes, prepared.right_ranks, - prepared.component_count); + prepared.component_count, prepared.communicator_rank); if (prepared.distributed) { - if (all_reduce_sum(layouts_match ? 0L : 1L) != 0) + if (all_reduce_sum(layouts_match ? 0L : 1L, prepared.communicator) != 0) throw std::runtime_error( "multi-block interface runtime fields differ from their prepared layouts on one " "or more MPI ranks"); @@ -842,7 +905,7 @@ class InterfaceFluxScheduler { } } if (prepared.distributed) - all_reduce_sum_inplace(prepared.traces.data(), prepared.traces.size()); + all_reduce_sum_inplace(prepared.traces.data(), prepared.traces.size(), prepared.communicator); const InterfaceFluxBatch batch{left, right, prepared.flux.data(), prepared.face_count, prepared.component_count}; @@ -853,7 +916,7 @@ class InterfaceFluxScheduler { evaluator_failure = std::current_exception(); } if (prepared.distributed) { - if (all_reduce_sum(evaluator_failure ? 1L : 0L) != 0) + if (all_reduce_sum(evaluator_failure ? 1L : 0L, prepared.communicator) != 0) throw std::runtime_error("multi-block interface evaluator failed on one or more MPI ranks"); } else if (evaluator_failure) { std::rethrow_exception(evaluator_failure); @@ -862,10 +925,10 @@ class InterfaceFluxScheduler { for (const Real value : prepared.flux) finite_flux = finite_flux && std::isfinite(static_cast(value)); if (prepared.distributed) { - if (all_reduce_sum(finite_flux ? 0L : 1L) != 0) + if (all_reduce_sum(finite_flux ? 0L : 1L, prepared.communicator) != 0) throw std::runtime_error( "multi-block interface evaluator returned a non-finite flux on one or more MPI ranks"); - require_distributed_flux_consensus_(prepared.flux, prepared.consensus); + require_distributed_flux_consensus_(prepared.flux, prepared.consensus, prepared.communicator); } else if (!finite_flux) { throw std::runtime_error("multi-block interface evaluator returned a non-finite flux"); } From 518ecaabd59249047b3e2b2f4de0a8ef7bbfb1aa Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:26:47 +0200 Subject: [PATCH 007/109] refactor(runtime): scope layout transfers to execution communicator --- src/runtime/system/system_layout_transfer.cpp | 99 +++++++++++-------- 1 file changed, 58 insertions(+), 41 deletions(-) diff --git a/src/runtime/system/system_layout_transfer.cpp b/src/runtime/system/system_layout_transfer.cpp index f6385c649..c6155e3b5 100644 --- a/src/runtime/system/system_layout_transfer.cpp +++ b/src/runtime/system/system_layout_transfer.cpp @@ -79,8 +79,8 @@ PopsExecutionContextV1 execution_view(const SystemLayoutTransferExecution& execu execution.communicator_datatype_identity.c_str()}; } -void validate_world_execution(const SystemLayoutTransferExecution& execution, - const CommunicatorView& world) { +CommunicatorView resolve_execution_communicator(const SystemLayoutTransferExecution& execution, + const CommunicatorView& field_rank_space) { const PopsExecutionContextV1 view = execution_view(execution); component::validate_execution_context(view); if (execution.memory_space != POPS_MEMORY_SPACE_HOST_V1 && @@ -88,43 +88,54 @@ void validate_world_execution(const SystemLayoutTransferExecution& execution, throw std::invalid_argument( "prepared System layout transfer requires host-addressable native field storage"); if (execution.communicator_identity == "serial") { - if (world.active()) + if (field_rank_space.active()) throw std::invalid_argument( "serial layout-transfer execution requires native MPI to be inactive"); - return; + return CommunicatorView{}; } - if (execution.communicator_identity != "MPI_COMM_WORLD") + if (execution.communicator_identity == POPS_EXECUTION_NONCOLLECTIVE_IDENTITY_V1) throw std::invalid_argument( - "prepared System layout transfer supports serial or exact MPI_COMM_WORLD execution"); + "prepared System layout transfer requires collective execution authority"); #ifdef POPS_HAS_MPI - if (!world.active()) + if (!field_rank_space.active()) throw std::invalid_argument( - "MPI_COMM_WORLD layout-transfer execution requires initialized native MPI"); - if (execution.communicator_f_handle != static_cast(MPI_Comm_c2f(MPI_COMM_WORLD)) || - execution.communicator_datatype_f_handle != - static_cast(MPI_Type_c2f(MPI_DOUBLE)) || + "collective layout-transfer execution requires initialized native MPI"); + const MPI_Comm communicator = + MPI_Comm_f2c(static_cast(execution.communicator_f_handle)); + if (communicator == MPI_COMM_NULL || + MPI_Type_f2c(static_cast(execution.communicator_datatype_f_handle)) != MPI_DOUBLE || execution.communicator_datatype_identity != "MPI_DOUBLE") throw std::invalid_argument( - "layout-transfer execution handles are not exact MPI_COMM_WORLD/MPI_DOUBLE authorities"); + "layout-transfer execution handles do not identify a live communicator/MPI_DOUBLE " + "authority"); + int relation = MPI_UNEQUAL; + ::pops::detail::require_mpi_success( + MPI_Comm_compare(communicator, field_rank_space.native_handle(), &relation), + "MPI_Comm_compare(layout-transfer field rank space)"); + if (relation != MPI_IDENT && relation != MPI_CONGRUENT) + throw std::invalid_argument( + "layout-transfer execution communicator must preserve the field rank space"); + return CommunicatorView{communicator}; #else - (void)world; + (void)field_rank_space; throw std::invalid_argument( - "MPI_COMM_WORLD layout-transfer execution requires an MPI-enabled PoPS build"); + "collective layout-transfer execution requires an MPI-enabled PoPS build"); #endif } template -void collectively_validate(const CommunicatorView& world, const char* where, Function&& function) { +void collectively_validate(const CommunicatorView& communicator, const char* where, + Function&& function) { std::exception_ptr failure; try { std::forward(function)(); } catch (...) { failure = std::current_exception(); } - const long failures = all_reduce_sum(failure ? 1L : 0L, world); + const long failures = all_reduce_sum(failure ? 1L : 0L, communicator); if (failures == 0) return; - if (world.size() == 1 && failure) + if (communicator.size() == 1 && failure) std::rethrow_exception(failure); throw std::runtime_error(std::string(where) + " failed on at least one MPI rank"); } @@ -172,14 +183,14 @@ std::uint64_t checked_elements(const Box2D& box, int components) { return static_cast(cells) * static_cast(components); } -std::uint64_t collective_elements(std::uint64_t local, const CommunicatorView& world) { - const auto ranks = static_cast(world.size()); +std::uint64_t collective_elements(std::uint64_t local, const CommunicatorView& communicator) { + const auto ranks = static_cast(communicator.size()); const std::uint64_t per_rank_limit = static_cast(std::numeric_limits::max()) / ranks; - const long invalid = all_reduce_max(local > per_rank_limit ? 1L : 0L, world); + const long invalid = all_reduce_max(local > per_rank_limit ? 1L : 0L, communicator); if (invalid != 0) throw std::overflow_error("layout-transfer global element count exceeds MPI long capacity"); - const long global = all_reduce_sum(static_cast(local), world); + const long global = all_reduce_sum(static_cast(local), communicator); return static_cast(global); } @@ -196,7 +207,7 @@ struct PreparedSystemLayoutTransfer::Impl { SystemLayoutTransferSpec spec; SystemLayoutTransferExecution execution; PopsExecutionContextV1 execution_abi{}; - CommunicatorView world; + CommunicatorView communicator; int source_block_index = -1; int target_block_index = -1; int components = 0; @@ -211,7 +222,8 @@ struct PreparedSystemLayoutTransfer::Impl { Impl(System& source_system, System& target_system, std::shared_ptr loaded, SystemLayoutTransferSpec transfer_spec, - SystemLayoutTransferExecution transfer_execution) + SystemLayoutTransferExecution transfer_execution, + const CommunicatorView& transfer_communicator) : source_owner(&source_system), target_owner(&target_system), source(source_system.p_.get()), @@ -220,7 +232,7 @@ struct PreparedSystemLayoutTransfer::Impl { spec(std::move(transfer_spec)), execution(std::move(transfer_execution)), execution_abi(execution_view(execution)), - world(world_communicator_view()) { + communicator(transfer_communicator) { validate_static_contract(); source_block_index = source->blocks_.index(spec.source_block); target_block_index = target->blocks_.index(spec.target_block); @@ -299,7 +311,6 @@ struct PreparedSystemLayoutTransfer::Impl { if (source_owner->lifecycle_state() == "assembling" || target_owner->lifecycle_state() == "assembling") throw std::invalid_argument("prepared System transfer requires bound native Systems"); - validate_world_execution(execution, world); const PopsComponentApiV1& api = component_handle->api(); if (api.component_id == nullptr || api.manifest_identity == nullptr || api.semantic_identity == nullptr || api.catalog_sha256 == nullptr || @@ -381,23 +392,28 @@ PreparedSystemLayoutTransfer::~PreparedSystemLayoutTransfer() = default; std::shared_ptr PreparedSystemLayoutTransfer::prepare( System& source, System& target, std::shared_ptr component, SystemLayoutTransferSpec spec, SystemLayoutTransferExecution execution) { - const CommunicatorView world = world_communicator_view(); + const CommunicatorView field_rank_space = world_communicator_view(); + CommunicatorView communicator; + collectively_validate(field_rank_space, "layout-transfer execution communicator", [&] { + communicator = resolve_execution_communicator(execution, field_rank_space); + }); std::unique_ptr pending; - collectively_validate(world, "prepared System layout-transfer allocation", [&] { + collectively_validate(communicator, "prepared System layout-transfer allocation", [&] { pending = std::make_unique(source, target, std::move(component), std::move(spec), - std::move(execution)); + std::move(execution), communicator); }); const std::string payload = pending->consensus_payload(); if (!all_ranks_agree_exact_ordered_byte_pairs({{"prepared-system-layout-transfer-v1", payload}}, - world)) + communicator)) throw std::invalid_argument( "prepared System layout-transfer contract differs between MPI ranks"); - collectively_validate(world, "native Transfer provider preparation", + collectively_validate(communicator, "native Transfer provider preparation", [&] { pending->prepare_provider(); }); // Warm the persistent copy schedule and MPI buffers before the first run step. This copy is // observationally inert: the carrier is private until capture() authenticates an attempt. - collectively_validate(world, "prepared System layout-transfer warmup", - [&] { parallel_copy(pending->source_snapshot, pending->source_state()); }); + collectively_validate(communicator, "prepared System layout-transfer warmup", [&] { + parallel_copy(pending->source_snapshot, pending->source_state(), communicator); + }); return std::shared_ptr( new PreparedSystemLayoutTransfer(std::move(pending))); } @@ -407,7 +423,7 @@ const SystemLayoutTransferSpec& PreparedSystemLayoutTransfer::spec() const noexc } void PreparedSystemLayoutTransfer::begin_transaction(std::uint64_t generation) { - collectively_validate(p_->world, "layout-transfer begin", [&] { + collectively_validate(p_->communicator, "layout-transfer begin", [&] { if (p_->active) throw std::logic_error("layout-transfer transaction is already active"); if (generation == 0 || generation <= p_->last_generation) @@ -425,7 +441,7 @@ void PreparedSystemLayoutTransfer::begin_transaction(std::uint64_t generation) { } void PreparedSystemLayoutTransfer::capture(std::uint64_t generation, std::uint64_t attempt) { - collectively_validate(p_->world, "layout-transfer capture", [&] { + collectively_validate(p_->communicator, "layout-transfer capture", [&] { p_->validate_active(generation, attempt, "layout-transfer capture"); if (p_->applied) throw std::logic_error( @@ -433,14 +449,15 @@ void PreparedSystemLayoutTransfer::capture(std::uint64_t generation, std::uint64 if (p_->captured_attempt != 0 && p_->captured_attempt != attempt) throw std::logic_error("layout-transfer source was already captured for another attempt"); }); - collectively_validate(p_->world, "layout-transfer source capture", - [&] { parallel_copy(p_->source_snapshot, p_->source_state()); }); + collectively_validate(p_->communicator, "layout-transfer source capture", [&] { + parallel_copy(p_->source_snapshot, p_->source_state(), p_->communicator); + }); p_->captured_attempt = attempt; } SystemLayoutTransferReceipt PreparedSystemLayoutTransfer::apply(std::uint64_t generation, std::uint64_t attempt) { - collectively_validate(p_->world, "layout-transfer apply preflight", [&] { + collectively_validate(p_->communicator, "layout-transfer apply preflight", [&] { p_->validate_active(generation, attempt, "layout-transfer apply"); if (p_->captured_attempt != attempt) throw std::logic_error("layout-transfer apply requires the exact captured attempt"); @@ -450,7 +467,7 @@ SystemLayoutTransferReceipt PreparedSystemLayoutTransfer::apply(std::uint64_t ge std::uint64_t local_source_elements = 0; std::uint64_t local_target_elements = 0; - collectively_validate(p_->world, "native Transfer apply", [&] { + collectively_validate(p_->communicator, "native Transfer apply", [&] { MultiFab& destination = p_->target_state(); try { for (int local = 0; local < p_->source_snapshot.local_size(); ++local) { @@ -549,13 +566,13 @@ SystemLayoutTransferReceipt PreparedSystemLayoutTransfer::apply(std::uint64_t ge receipt.operation = p_->spec.operation; receipt.generation = generation; receipt.attempt = attempt; - receipt.source_element_count = collective_elements(local_source_elements, p_->world); - receipt.destination_element_count = collective_elements(local_target_elements, p_->world); + receipt.source_element_count = collective_elements(local_source_elements, p_->communicator); + receipt.destination_element_count = collective_elements(local_target_elements, p_->communicator); return receipt; } void PreparedSystemLayoutTransfer::reject_attempt(std::uint64_t generation, std::uint64_t attempt) { - collectively_validate(p_->world, "layout-transfer rejected-attempt reset", [&] { + collectively_validate(p_->communicator, "layout-transfer rejected-attempt reset", [&] { p_->validate_active(generation, attempt, "layout-transfer rejected-attempt reset"); if (p_->captured_attempt != attempt) throw std::logic_error( From 3c428f065a7619155b5a43b6e446ca8972bbc58d Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:26:51 +0200 Subject: [PATCH 008/109] test(runtime): fence layout transfer communicator scope --- ...stem_layout_transfer_communicator_fence.py | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 tests/python/architecture/test_system_layout_transfer_communicator_fence.py diff --git a/tests/python/architecture/test_system_layout_transfer_communicator_fence.py b/tests/python/architecture/test_system_layout_transfer_communicator_fence.py new file mode 100644 index 000000000..f7da3877a --- /dev/null +++ b/tests/python/architecture/test_system_layout_transfer_communicator_fence.py @@ -0,0 +1,49 @@ +"""ADC-683 fences for execution-owned System layout-transfer collectives.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SOURCE = ROOT / "src/runtime/system/system_layout_transfer.cpp" + + +def _function(source: str, signature: str) -> str: + start = source.index(signature) + opening_brace = source.index("{", start) + depth = 0 + for offset in range(opening_brace, len(source)): + token = source[offset] + if token == "{": + depth += 1 + elif token == "}": + depth -= 1 + if depth == 0: + return source[start : offset + 1] + raise AssertionError(f"unterminated C++ function {signature}") + + +def test_layout_transfer_accepts_a_live_world_congruent_execution_context(): + source = SOURCE.read_text(encoding="utf-8") + resolver = _function(source, "CommunicatorView resolve_execution_communicator(") + + assert "MPI_COMM_WORLD" not in source + assert "MPI_Comm_f2c" in resolver + assert "MPI_Comm_compare(communicator, field_rank_space.native_handle()" in resolver + assert "relation != MPI_IDENT && relation != MPI_CONGRUENT" in resolver + assert "POPS_EXECUTION_NONCOLLECTIVE_IDENTITY_V1" in resolver + assert "return CommunicatorView{communicator};" in resolver + + +def test_layout_transfer_retains_the_resolved_context_for_every_hot_collective(): + source = SOURCE.read_text(encoding="utf-8") + implementation = source.split("struct PreparedSystemLayoutTransfer::Impl", maxsplit=1)[1] + hot_path = source.split("void PreparedSystemLayoutTransfer::begin_transaction", maxsplit=1)[1] + + assert source.count("world_communicator_view()") == 1 + assert "CommunicatorView communicator;" in implementation + assert "CommunicatorView world;" not in implementation + assert "p_->world" not in hot_path + assert "world_communicator_view()" not in hot_path + assert "parallel_copy(p_->source_snapshot, p_->source_state(), p_->communicator)" in hot_path + assert "collective_elements(local_source_elements, p_->communicator)" in hot_path + assert "collective_elements(local_target_elements, p_->communicator)" in hot_path From 9a25359b02aff282205dc490c1f75fc6812b32ee Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:42:39 +0200 Subject: [PATCH 009/109] feat(diagnostics): author typed balance ledgers --- python/pops/diagnostics/__init__.py | 6 +- python/pops/diagnostics/balance.py | 87 ++++++++++++++++++++++++ python/pops/diagnostics/measures.py | 69 ++++++++++++++++++- python/pops/time/_program/contract.py | 10 +++ python/pops/time/_program/diagnostics.py | 79 +++++++++++++++++++++ 5 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 python/pops/diagnostics/balance.py diff --git a/python/pops/diagnostics/__init__.py b/python/pops/diagnostics/__init__.py index 76a1ac342..f8b3e6296 100644 --- a/python/pops/diagnostics/__init__.py +++ b/python/pops/diagnostics/__init__.py @@ -3,9 +3,11 @@ Historical lowercase descriptor factories are intentionally absent: diagnostics are authored with immutable typed measures and attached to the Case consumer graph. """ +from .balance import BalanceLedger from .invariants import invariants -from .measures import ConservationCheck, Integral, MinMax, Norm, StepChangeNorm +from .measures import Balance, ConservationCheck, Integral, MinMax, Norm, StepChangeNorm __all__ = [ - "ConservationCheck", "Integral", "MinMax", "Norm", "StepChangeNorm", "invariants", + "Balance", "BalanceLedger", "ConservationCheck", "Integral", "MinMax", "Norm", + "StepChangeNorm", "invariants", ] diff --git a/python/pops/diagnostics/balance.py b/python/pops/diagnostics/balance.py new file mode 100644 index 000000000..5b9bfd173 --- /dev/null +++ b/python/pops/diagnostics/balance.py @@ -0,0 +1,87 @@ +"""Typed identity shared by native Program balance evidence and output consumers.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from pops.identity import Identity, make_identity + + +BALANCE_TERM_NAMES = ( + "storage_change", + "outward_boundary_flux", + "sources", + "reflux", + "projection", +) + + +def _canonical_name(value: Any, *, where: str) -> str: + if not isinstance(value, str) or not value or value.strip() != value: + raise TypeError("%s must be non-empty canonical text" % where) + return value + + +@dataclass(frozen=True, slots=True) +class BalanceLedger: + """Identity joining one Program-authored discrete balance to one consumer. + + The ledger does not contain values. :meth:`Program.record_balance` writes the five + reduced scalars into the current native step-attempt mailbox, while + :class:`pops.diagnostics.Balance` selects the same identity after that attempt has + advanced successfully. + """ + + name: str + identity: Identity = field(init=False) + __pops_ir_immutable__ = True + + def __post_init__(self) -> None: + name = _canonical_name(self.name, where="BalanceLedger.name") + object.__setattr__(self, "name", name) + object.__setattr__( + self, + "identity", + make_identity("balance-ledger", {"schema_version": 1, "name": name}), + ) + + def to_data(self) -> dict[str, Any]: + return { + "schema_version": 1, + "name": self.name, + "identity": self.identity.to_data(), + } + + def route_identity(self, block: Any) -> Identity: + from pops.problem.handles import BlockHandle + + if not isinstance(block, BlockHandle): + raise TypeError("balance ledger block must be a BlockHandle") + return make_identity( + "balance-ledger-route", + { + "schema_version": 1, + "ledger": self.identity.to_data(), + # The Program records this route before Case resolution, whereas the + # consumer is resolved later. Runtime block names are unique inside one + # Case/Program, and the consumer quantity separately carries the complete + # canonical block/state identity. + "runtime_block": block.local_id, + }, + ) + + +def balance_record_name(route: Any, term: Any) -> str: + """Return the reserved native Program diagnostic key for one exact term.""" + if ( + type(route) is not Identity + or route.domain != "balance-ledger-route" + or route.schema_version != 1 + ): + raise TypeError("balance route must be an exact balance-ledger-route Identity") + if term not in BALANCE_TERM_NAMES: + raise ValueError("unknown balance term %r" % (term,)) + return "pops.balance-term.v1:%s:%s" % (route.token, term) + + +__all__ = ["BALANCE_TERM_NAMES", "BalanceLedger"] diff --git a/python/pops/diagnostics/measures.py b/python/pops/diagnostics/measures.py index 138fbffa2..bf4c36cf8 100644 --- a/python/pops/diagnostics/measures.py +++ b/python/pops/diagnostics/measures.py @@ -2,7 +2,7 @@ Spec 5 names a diagnostic with a TYPED object, not the string form ``diagnostics.norm(kind="l2")``. :class:`Norm` / :class:`Integral` / :class:`MinMax` / -:class:`ConservationCheck` are those objects -- inert descriptors that DESCRIBE a scalar +:class:`Balance` / :class:`ConservationCheck` are those objects -- inert descriptors that DESCRIBE a scalar reduction over a block (and an optional model role): the reduction kind, whether it needs an MPI reduction, its cadence slot and its AMR / multi-level compatibility, all carried as METADATA. They compute nothing; the C++ / Kokkos / MPI runtime evaluates the reduction. @@ -23,6 +23,8 @@ from pops.descriptors import Availability, Descriptor from pops.linalg.norms import _Norm +from .balance import BalanceLedger + def _ref_name(value: Any) -> Any: """The stable display name for a block / role reference (its ``name`` or its repr). @@ -304,6 +306,57 @@ def diagnostic_execution(self) -> dict[str, Any]: } +class Balance(_Measure): + """Accepted five-term discrete balance produced by the native time Program. + + ``Balance`` never reconstructs terms from output arrays. The matching + :class:`BalanceLedger` must be populated with ``Program.record_balance`` during + the same native attempt. The runtime then consumes exactly storage change, + outward boundary flux, sources, reflux and projection while its accepted-state + transaction still retains the pre-step image. The residual convention is storage + change plus outward flux, minus sources, reflux and projection. + """ + + category = "diagnostic_balance" + scheme = "discrete_balance" + reduction = "accepted_balance" + + def __init__( + self, + ledger: Any, + *, + block: Any, + cadence: Any = None, + ) -> None: + if type(ledger) is not BalanceLedger: + raise TypeError( + "Balance(ledger=...) requires an exact pops.diagnostics.BalanceLedger" + ) + if block is None: + raise TypeError("Balance(block=...) requires an exact physics BlockHandle") + super().__init__(block=block, role=None, cadence=cadence) + self.ledger = ledger + + def options(self) -> dict: + options = super().options() + options["ledger"] = self.ledger.to_data() + return options + + def diagnostic_execution(self) -> dict[str, Any]: + route = self.ledger.route_identity(self.block) + return { + "schema_version": 1, + "role": None, + "operations": [ + { + **_operation("balance", "accepted_balance"), + "balance_route": route.token, + }, + ], + "conservation": None, + } + + class ConservationCheck(Descriptor): """A typed conservation check on a diagnostic quantity: ``ConservationCheck(Integral(...))``. @@ -380,6 +433,11 @@ def diagnostic_execution(self) -> dict[str, Any]: raise ValueError( "ConservationCheck requires one scalar diagnostic quantity; " "a multi-valued MinMax check is ambiguous") + if operations[0].get("reduction") == "accepted_balance": + raise ValueError( + "ConservationCheck cannot wrap an open-domain Balance; inspect its explicit " + "five-term residual instead" + ) return { "schema_version": 1, "role": plan.get("role"), @@ -425,4 +483,11 @@ def inspect(self) -> Any: return info -__all__ = ["Norm", "Integral", "MinMax", "ConservationCheck"] +__all__ = [ + "Balance", + "Norm", + "Integral", + "MinMax", + "ConservationCheck", + "StepChangeNorm", +] diff --git a/python/pops/time/_program/contract.py b/python/pops/time/_program/contract.py index 1bc5ab26d..ff3a528a6 100644 --- a/python/pops/time/_program/contract.py +++ b/python/pops/time/_program/contract.py @@ -198,6 +198,16 @@ def subcycle(self, state: Any, *, clock: Any, within: Any, def _compare(self, lhs: Any, rhs: Any, cmp: Any) -> Any: ... def _scalar_binop(self, a: Any, b: Any, fn: Any) -> Any: ... def record_scalar(self, name: Any, value: Any) -> Any: ... + def record_balance( + self, + ledger: Any, + *, + storage_change: Any, + outward_boundary_flux: Any, + sources: Any, + reflux: Any, + projection: Any, + ) -> tuple[Any, ...]: ... # --- solve / commit / board sugar (_ProgramSolve) --- def _solve_linear(self, *, operator: Any, rhs: Any, prepared: Any, properties: Any, diff --git a/python/pops/time/_program/diagnostics.py b/python/pops/time/_program/diagnostics.py index 86f61b4c5..b2cee7644 100644 --- a/python/pops/time/_program/diagnostics.py +++ b/python/pops/time/_program/diagnostics.py @@ -26,6 +26,85 @@ def record(self, name: Any, value: Any) -> ProgramValue: % (name, value)) return self.record_scalar(name, value) + @atomic_authoring + def record_balance( + self, + ledger: Any, + *, + storage_change: Any, + outward_boundary_flux: Any, + sources: Any, + reflux: Any, + projection: Any, + ) -> tuple[ProgramValue, ...]: + """Publish one exact five-term balance into the current native attempt. + + Every term is a signed, time-integrated increment for this Program invocation and + must be an additive global Program reduction (sum/dot), or scalar arithmetic composed + exclusively from such reductions and exact literals. The native mailbox accumulates + these increments across cadence substeps in the same public macro-step. Raw Python values, + extrema/norm reductions, and rank-local runtime scalars are rejected. The five records are + attempt-local: a rejected step or consumer rollback cannot leave evidence for a later sample. + """ + from pops.diagnostics.balance import ( + BALANCE_TERM_NAMES, + BalanceLedger, + balance_record_name, + ) + + if type(ledger) is not BalanceLedger: + raise TypeError( + "record_balance ledger must be an exact pops.diagnostics.BalanceLedger" + ) + supplied = { + "storage_change": storage_change, + "outward_boundary_flux": outward_boundary_flux, + "sources": sources, + "reflux": reflux, + "projection": projection, + } + + def require_reduced(value: Any, term: str, seen: set[int]) -> ProgramValue: + value = self._canonical_value(value) + if not isinstance(value, ProgramValue) or value.prog is not self \ + or value.vtype != "scalar": + raise TypeError( + "record_balance %s must be a scalar from this Program" % term + ) + if value.id in seen: + return value + seen.add(value.id) + if value.op == "reduce": + if value.attrs.get("kind") not in {"sum", "dot"}: + raise ValueError( + "record_balance %s requires additive sum/dot reductions; got %r" + % (term, value.attrs.get("kind")) + ) + return value + if value.op == "scalar_op" and value.inputs: + for item in value.inputs: + require_reduced(item, term, seen) + return value + raise ValueError( + "record_balance %s must be a global reduction or arithmetic composed " + "only from global reductions; got scalar op %r" % (term, value.op) + ) + + terms = { + name: require_reduced(supplied[name], name, set()) + for name in BALANCE_TERM_NAMES + } + blocks = {value.block for value in terms.values()} + if None in blocks or len(blocks) != 1: + raise ValueError( + "record_balance terms must reduce one exact common physics block" + ) + route = ledger.route_identity(next(iter(blocks))) + return tuple( + self.record_scalar(balance_record_name(route, name), terms[name]) + for name in BALANCE_TERM_NAMES + ) + @atomic_authoring def check_invariant(self, name: Any, before: Any = None, after: Any = None, tolerance: Any = 1e-10) -> ProgramValue: From 6b4612f14cb2e2d660c61677be9cd09e16fc6265 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:42:58 +0200 Subject: [PATCH 010/109] feat(runtime): publish accepted five-term balances --- include/pops/runtime/amr_system.hpp | 3 + .../runtime/program/program_runtime_state.hpp | 63 ++++++++++++++- include/pops/runtime/system.hpp | 3 + python/bindings/core/init/init_amr.cpp | 1 + python/bindings/core/init/init_system.cpp | 1 + python/pops/_pops.pyi | 2 + python/pops/output/_consumer_contracts.py | 44 +++++++++-- python/pops/runtime/_runtime_consumers.py | 79 ++++++++++++++++++- src/runtime/amr/amr_system.cpp | 10 +++ src/runtime/system/system_impl.hpp | 3 + src/runtime/system/system_program.cpp | 6 ++ 11 files changed, 202 insertions(+), 13 deletions(-) diff --git a/include/pops/runtime/amr_system.hpp b/include/pops/runtime/amr_system.hpp index 06d82e92a..ac30577f9 100644 --- a/include/pops/runtime/amr_system.hpp +++ b/include/pops/runtime/amr_system.hpp @@ -890,6 +890,9 @@ class AmrSystem { /// The recorded diagnostic @p name (0 if absent) / the whole map. Exposed to Python for inspection. POPS_EXPORT double program_diagnostic(const std::string& name) const; POPS_EXPORT std::map program_diagnostics() const; + /// Five current-attempt scalars for one typed balance route. RuntimeInstance calls this only + /// inside its active outer accepted-step transaction; missing/stale/non-finite evidence fails. + POPS_EXPORT std::map accepted_balance_terms(const std::string& route) const; POPS_EXPORT void begin_step_projection_report(); POPS_EXPORT void note_step_projection(const std::string& name); POPS_EXPORT std::vector consume_step_projections(); diff --git a/include/pops/runtime/program/program_runtime_state.hpp b/include/pops/runtime/program/program_runtime_state.hpp index e884f0bf0..31a74501a 100644 --- a/include/pops/runtime/program/program_runtime_state.hpp +++ b/include/pops/runtime/program/program_runtime_state.hpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -257,6 +258,11 @@ struct ProgramRuntimeState { /// COMPILED-PROGRAM SCALAR DIAGNOSTICS (ADC-414): name -> last value recorded via P.record_scalar. /// Lives here (not the .so) so it outlives the step closure and Python can read it. Used by BOTH. std::map diagnostics_; + /// Reserved balance records for the current native attempt only. Unlike diagnostics_, this + /// mailbox is cleared before every public step and is never checkpointed. Accepted balance + /// consumers read it while the facade's outer transaction still retains U^n, so a missing term + /// cannot silently reuse the preceding step. + std::map step_balance_terms_; /// Attempt-local identities of ProjectAndRecheck branches that actually executed. This report /// mailbox is cleared at attempt entry and consumed by the Python transaction coordinator before /// commit or rollback; it is deliberately not checkpoint or accepted scientific state. @@ -672,9 +678,21 @@ struct ProgramRuntimeState { " set_clock cannot reuse an active stride window; restore its strict checkpoint image"); } - /// Record a compiled-Program scalar diagnostic (ADC-414): the installed Program writes named scalars - /// via P.record_scalar; Python reads them after the step. Idempotent (last write wins). - void record_diagnostic(const std::string& name, Real value) { diagnostics_[name] = value; } + /// Record a compiled-Program scalar. Ordinary P.record_scalar names remain inspectable after the + /// step with last-write-wins semantics. The reserved balance prefix is attempt-local and additive. + void record_diagnostic(const std::string& name, Real value) { + // A Program cadence may invoke the compiled body several times inside one public macro-step. + // Balance records are signed, time-integrated increments and therefore accumulate across those + // invocations. Ordinary inspection diagnostics retain their historical last-write-wins contract. + static constexpr const char* kBalancePrefix = "pops.balance-term.v1:"; + if (name.rfind(kBalancePrefix, 0) == 0) { + auto [entry, inserted] = step_balance_terms_.try_emplace(name, value); + if (!inserted) + entry->second += value; + return; + } + diagnostics_[name] = value; + } /// Read the named diagnostic, FAIL-LOUD if the Program never recorded it. @p runtime names the /// Program subsystem setter in the message (not a generic getter). @throws std::out_of_range. @@ -690,7 +708,44 @@ struct ProgramRuntimeState { /// The whole name -> value diagnostics map (checkpoint / inspection). By value: inert copy. std::map diagnostics() const { return diagnostics_; } - void begin_step_projection_report() { step_projections_.clear(); } + void begin_step_projection_report() { + step_projections_.clear(); + step_balance_terms_.clear(); + } + + /// Return exactly the five native Program scalars recorded for one typed balance route during the + /// current attempt. The facade separately proves that an external accepted-step transaction is + /// active. No zero, stale value, or array-derived Python fallback is permitted. + std::map accepted_balance_terms(const std::string& route, + const std::string& runtime) const { + static constexpr const char* kRoutePrefix = "pops.balance-ledger-route.v1:sha256:"; + static constexpr std::array kTerms{"storage_change", "outward_boundary_flux", + "sources", "reflux", "projection"}; + const std::string prefix{kRoutePrefix}; + if (route.size() != prefix.size() + 64 || route.compare(0, prefix.size(), prefix) != 0 || + !std::all_of(route.begin() + static_cast(prefix.size()), route.end(), + [](unsigned char value) { + return (value >= '0' && value <= '9') || (value >= 'a' && value <= 'f'); + })) + throw std::invalid_argument( + runtime + "::_accepted_balance_terms requires a canonical balance-ledger-route identity"); + std::map result; + for (const char* term : kTerms) { + const std::string record = "pops.balance-term.v1:" + route + ":" + term; + const auto found = step_balance_terms_.find(record); + if (found == step_balance_terms_.end()) + throw std::runtime_error( + runtime + "::_accepted_balance_terms: current native attempt omitted term '" + term + + "'; Program.record_balance must publish all five terms"); + if (!std::isfinite(static_cast(found->second))) + throw std::runtime_error( + runtime + + "::_accepted_balance_terms: current native attempt produced non-finite term '" + term + + "'"); + result.emplace(term, found->second); + } + return result; + } void note_step_projection(const std::string& name) { if (name.empty()) diff --git a/include/pops/runtime/system.hpp b/include/pops/runtime/system.hpp index b0432da83..c427ee4ae 100644 --- a/include/pops/runtime/system.hpp +++ b/include/pops/runtime/system.hpp @@ -1236,6 +1236,9 @@ class System { /// All recorded diagnostics (name -> last recorded value). Empty when the program records none. /// Exposed to Python as sim.program_diagnostics() (a dict); program_diagnostic(name) reads one. POPS_EXPORT std::map program_diagnostics() const; + /// Five current-attempt scalars for one typed balance route. RuntimeInstance calls this only + /// inside its active outer accepted-step transaction; missing/stale/non-finite evidence fails. + POPS_EXPORT std::map accepted_balance_terms(const std::string& route) const; POPS_EXPORT void begin_step_projection_report(); POPS_EXPORT void note_step_projection(const std::string& name); POPS_EXPORT std::vector consume_step_projections(); diff --git a/python/bindings/core/init/init_amr.cpp b/python/bindings/core/init/init_amr.cpp index 54703525f..f1f2edda0 100644 --- a/python/bindings/core/init/init_amr.cpp +++ b/python/bindings/core/init/init_amr.cpp @@ -824,6 +824,7 @@ void bind_amr_program(py::class_& cls) { // driver records a measured scalar into each cadence tick. .def("program_diagnostic", &AmrSystem::program_diagnostic, py::arg("name")) .def("program_diagnostics", &AmrSystem::program_diagnostics) + .def("_accepted_balance_terms", &AmrSystem::accepted_balance_terms, py::arg("route")) .def("_consume_step_projections", &AmrSystem::consume_step_projections) .def("record_program_diagnostic", &AmrSystem::record_program_diagnostic, py::arg("name"), py::arg("value")) diff --git a/python/bindings/core/init/init_system.cpp b/python/bindings/core/init/init_system.cpp index b77a53d46..fa71bbfc9 100644 --- a/python/bindings/core/init/init_system.cpp +++ b/python/bindings/core/init/init_system.cpp @@ -354,6 +354,7 @@ void bind_system_program(py::class_& cls) { // program_diagnostics() returns the whole name -> value dict. .def("program_diagnostic", &System::program_diagnostic, py::arg("name")) .def("program_diagnostics", &System::program_diagnostics) + .def("_accepted_balance_terms", &System::accepted_balance_terms, py::arg("route")) .def("_consume_step_projections", &System::consume_step_projections) // ADC-542: the native collective reduction over a named block the diagnostics driver drives to // fire a declared typed measure (Norm / Integral / MinMax) each cadence tick, and the sink the diff --git a/python/pops/_pops.pyi b/python/pops/_pops.pyi index 10856fe97..5ea39b03c 100644 --- a/python/pops/_pops.pyi +++ b/python/pops/_pops.pyi @@ -267,6 +267,7 @@ class System: def __init__(self, config: SystemConfig) -> None: ... def solve_fields(self) -> _SolveReport: ... def _consume_step_projections(self) -> list[str]: ... + def _accepted_balance_terms(self, route: str) -> dict[str, float]: ... def output_state_local_pieces( self, block: str, level: int ) -> tuple[dict[str, object], ...]: ... @@ -286,6 +287,7 @@ class AmrSystem: def n_levels(self) -> int: ... def configured_n_levels(self) -> int: ... def _consume_step_projections(self) -> list[str]: ... + def _accepted_balance_terms(self, route: str) -> dict[str, float]: ... def materialize_program_restart_histories( self, payload: bytes, diff --git a/python/pops/output/_consumer_contracts.py b/python/pops/output/_consumer_contracts.py index cfe4260a5..3c8d7bc58 100644 --- a/python/pops/output/_consumer_contracts.py +++ b/python/pops/output/_consumer_contracts.py @@ -291,6 +291,7 @@ def to_data(self) -> dict[str, Any]: _DIAGNOSTIC_REDUCTIONS = frozenset({ "sum", "abs_sum", "sum_sq", "min", "max", "abs_max", "step_change_l2", + "accepted_balance", }) _DIAGNOSTIC_TRANSFORMS = frozenset({"identity", "sqrt"}) _DIAGNOSTIC_COLLECTIVES = { @@ -301,6 +302,9 @@ def to_data(self) -> dict[str, Any]: "max": "global_max", "abs_max": "global_max", "step_change_l2": "global_sum", + # The five Program scalars were already reduced while executing the native + # accepted attempt. Reading its mailbox adds no second consumer collective. + "accepted_balance": None, } @@ -320,11 +324,15 @@ def _diagnostic_execution(value: Any) -> Mapping[str, Any]: normalized = [] for index, operation in enumerate(operations): where = "DiagnosticQuantity.execution.operations[%d]" % index - if not isinstance(operation, Mapping) or set(operation) != { - "name", "reduction", "transform", "metric_weighted"}: + if not isinstance(operation, Mapping): + raise TypeError("%s has an unknown schema" % where) + reduction = operation.get("reduction") + expected = {"name", "reduction", "transform", "metric_weighted"} + if reduction == "accepted_balance": + expected.add("balance_route") + if set(operation) != expected: raise TypeError("%s has an unknown schema" % where) name = _text(operation["name"], "%s.name" % where) - reduction = operation["reduction"] if reduction not in _DIAGNOSTIC_REDUCTIONS: raise ValueError("%s.reduction is not a supported native reduction" % where) transform = operation["transform"] @@ -335,17 +343,42 @@ def _diagnostic_execution(value: Any) -> Mapping[str, Any]: raise TypeError("%s.metric_weighted must be an exact bool" % where) if weighted and reduction not in {"sum", "abs_sum", "sum_sq"}: raise ValueError("only additive diagnostic reductions may be metric-weighted") - normalized.append({ + row = { "name": name, "reduction": reduction, "transform": transform, "metric_weighted": weighted, - }) + } + if reduction == "accepted_balance": + if transform != "identity" or weighted: + raise ValueError( + "accepted balance evidence cannot apply a scalar transform or metric weight" + ) + route = Identity.from_token(operation["balance_route"]) + if route.domain != "balance-ledger-route" or route.schema_version != 1: + raise ValueError( + "accepted balance route must use the version-1 balance-ledger-route identity" + ) + row["balance_route"] = route.token + normalized.append(row) if len({row["name"] for row in normalized}) != len(normalized): raise ValueError("DiagnosticQuantity execution operation names must be unique") + has_accepted_balance = any( + row["reduction"] == "accepted_balance" for row in normalized + ) + if has_accepted_balance and len(normalized) != 1: + raise ValueError( + "accepted balance evidence must be the sole diagnostic execution operation" + ) + if has_accepted_balance and role is not None: + raise ValueError("accepted balance evidence cannot select one component role") conservation = value["conservation"] normalized_conservation = None if conservation is not None: + if has_accepted_balance: + raise ValueError( + "accepted open-domain balance evidence cannot declare an invariant tolerance" + ) if not isinstance(conservation, Mapping) or set(conservation) != {"tolerance"}: raise TypeError("DiagnosticQuantity.execution.conservation has an unknown schema") tolerance = _nonnegative_binary64_hex( @@ -367,6 +400,7 @@ def diagnostic_collective_operations(execution: Any) -> tuple[str, ...]: return tuple(sorted({ _DIAGNOSTIC_COLLECTIVES[operation["reduction"]] for operation in canonical["operations"] + if _DIAGNOSTIC_COLLECTIVES[operation["reduction"]] is not None })) diff --git a/python/pops/runtime/_runtime_consumers.py b/python/pops/runtime/_runtime_consumers.py index 0d7dbddd1..99c6d8f56 100644 --- a/python/pops/runtime/_runtime_consumers.py +++ b/python/pops/runtime/_runtime_consumers.py @@ -2411,6 +2411,30 @@ def _validate_diagnostic_providers(self) -> None: reductions = { operation["reduction"] for operation in quantity.execution["operations"] } + layout = layouts.get(quantity.layout_id) + if layout is None: + raise KeyError("diagnostic selected unknown layout %s" % quantity.layout_id) + engine = self._owner._executor_for_block(block) + if "accepted_balance" in reductions and reductions != {"accepted_balance"}: + raise ValueError( + "accepted balance evidence cannot be mixed with field reductions" + ) + if reductions == {"accepted_balance"}: + if len(quantity.execution["operations"]) != 1: + raise ValueError("accepted balance requires exactly one native evidence route") + if quantity.execution["role"] is not None: + raise ValueError("accepted balance route cannot carry a component role") + if not callable(getattr(engine, "_accepted_balance_terms", None)): + raise NotImplementedError( + "balance diagnostic requires native _accepted_balance_terms(route)" + ) + configured_levels = tuple(level.index for level in layout.levels) + if tuple(quantity.levels) != configured_levels: + raise ValueError( + "balance diagnostic must select the complete configured hierarchy; " + "a subset cannot be reconciled with the accepted Program ledger" + ) + continue if reductions == {"step_change_l2"}: if quantity.execution["role"] is not None: raise ValueError("step-change norm is a whole-state diagnostic") @@ -2422,10 +2446,6 @@ def _validate_diagnostic_providers(self) -> None: ) else: self._diagnostic_component(names, roles, quantity.execution["role"]) - layout = layouts.get(quantity.layout_id) - if layout is None: - raise KeyError("diagnostic selected unknown layout %s" % quantity.layout_id) - engine = self._owner._executor_for_block(block) if layout.adaptive: if not callable(getattr(engine, "composite_reduce", None)): raise NotImplementedError( @@ -2524,6 +2544,33 @@ def _native_diagnostic_reduction( kind = reduction + ("_all" if full_state else "") return float(cast(Any, native)(block, kind, component)), False + @staticmethod + def _native_balance_terms(engine: Any, route: str) -> Any: + """Read one current-attempt balance tuple from the native transaction mailbox.""" + from pops.output.diagnostics import BalanceTerms + + native = getattr(engine, "_accepted_balance_terms", None) + if not callable(native): + raise RuntimeError("installed runtime has no accepted balance evidence provider") + raw = native(route) + required = { + "storage_change", + "outward_boundary_flux", + "sources", + "reflux", + "projection", + } + if not isinstance(raw, Mapping) or set(raw) != required: + raise TypeError( + "native accepted balance provider must return exactly storage_change, " + "outward_boundary_flux, sources, reflux, and projection" + ) + if any(type(raw[name]) is not float for name in required): + raise TypeError( + "native accepted balance provider terms must be exact floating-point scalars" + ) + return BalanceTerms(**{name: raw[name] for name in sorted(required)}) + def _diagnostic_values( self, manifest: Any, @@ -2550,6 +2597,30 @@ def _diagnostic_values( variables, roles = _conservative_metadata(self._owner, block) execution = quantity.execution reductions = {operation["reduction"] for operation in execution["operations"]} + if reductions == {"accepted_balance"}: + if "accepted_balance" in skip_reductions: + continue + operation, = execution["operations"] + balance = self._native_balance_terms( + engine, operation["balance_route"]) + terms = { + "storage_change": balance.storage_change, + "outward_boundary_flux": balance.outward_boundary_flux, + "sources": balance.sources, + "reflux": balance.reflux, + "projection": balance.projection, + } + key = DiagnosticKey( + quantity.handle, + self._owner._component_manifests[block].manifest_digest, + self._owner.layout_identity(quantity.layout_id), + min(levels), + quantity.identity.token, + "discrete_balance", + ) + values.append(DiagnosticPayload( + key, balance.residual, "unspecified", terms)) + continue if reductions == {"step_change_l2"}: component, full_state = 0, True else: diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index 4a827ec8f..a9be4b1fa 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -457,6 +457,7 @@ struct AmrSystem::Impl { double cadence_clock_restore_accepted_time = 0.0; int cadence_clock_restore_macro_step = 0; std::map program_diagnostics; + std::map step_balance_terms; pops::runtime::program::CacheManager cache; pops::runtime::program::HistoryManager history; pops::runtime::program::Profiler profiler; @@ -501,6 +502,7 @@ struct AmrSystem::Impl { cadence_clock_restore_accepted_time = impl.program_.cadence_clock_restore_accepted_time_; cadence_clock_restore_macro_step = impl.program_.cadence_clock_restore_macro_step_; copy_value_map_into(program_diagnostics, impl.program_.diagnostics_); + copy_value_map_into(step_balance_terms, impl.program_.step_balance_terms_); // AMR currently owns its native cache/history rings inside AmrRuntime. These two shared // ProgramRuntimeState containers are therefore empty on the AMR path, but retain their value // contract so a future target can populate them without weakening rollback semantics. @@ -531,6 +533,7 @@ struct AmrSystem::Impl { impl.program_.cadence_clock_restore_accepted_time_ = cadence_clock_restore_accepted_time; impl.program_.cadence_clock_restore_macro_step_ = cadence_clock_restore_macro_step; copy_value_map_into(impl.program_.diagnostics_, program_diagnostics); + copy_value_map_into(impl.program_.step_balance_terms_, step_balance_terms); impl.program_.cache_ = cache; impl.program_.hist_ = history; impl.program_.profiler_ = profiler; @@ -3536,6 +3539,13 @@ double AmrSystem::program_diagnostic(const std::string& name) const { std::map AmrSystem::program_diagnostics() const { return p_->program_.diagnostics_; } +std::map AmrSystem::accepted_balance_terms(const std::string& route) const { + if (!p_->external_step_transaction_active_ || p_->external_step_transaction_committed_) + throw std::runtime_error( + "AmrSystem::_accepted_balance_terms requires an active uncommitted external step " + "transaction"); + return p_->program_.accepted_balance_terms(route, "AmrSystem"); +} void AmrSystem::begin_step_projection_report() { p_->program_.begin_step_projection_report(); } diff --git a/src/runtime/system/system_impl.hpp b/src/runtime/system/system_impl.hpp index 1e3cd8f43..aea9dc2d0 100644 --- a/src/runtime/system/system_impl.hpp +++ b/src/runtime/system/system_impl.hpp @@ -616,6 +616,7 @@ struct System::Impl { double cadence_clock_restore_accepted_time; int cadence_clock_restore_macro_step; std::map program_diagnostics; + std::map step_balance_terms; pops::runtime::program::CacheManager cache; pops::runtime::program::HistoryManager history; pops::runtime::program::Profiler profiler; @@ -639,6 +640,7 @@ struct System::Impl { cadence_clock_restore_accepted_time(impl.program_.cadence_clock_restore_accepted_time_), cadence_clock_restore_macro_step(impl.program_.cadence_clock_restore_macro_step_), program_diagnostics(impl.program_.diagnostics_), + step_balance_terms(impl.program_.step_balance_terms_), cache(impl.program_.cache_), history(impl.program_.hist_), profiler(impl.program_.profiler_), @@ -670,6 +672,7 @@ struct System::Impl { impl.program_.cadence_clock_restore_accepted_time_ = cadence_clock_restore_accepted_time; impl.program_.cadence_clock_restore_macro_step_ = cadence_clock_restore_macro_step; impl.program_.diagnostics_ = program_diagnostics; + impl.program_.step_balance_terms_ = step_balance_terms; impl.program_.cache_ = cache; impl.program_.hist_ = history; impl.program_.profiler_ = profiler; diff --git a/src/runtime/system/system_program.cpp b/src/runtime/system/system_program.cpp index a675fb6fe..43127668f 100644 --- a/src/runtime/system/system_program.cpp +++ b/src/runtime/system/system_program.cpp @@ -417,6 +417,12 @@ Real System::program_diagnostic(const std::string& name) const { std::map System::program_diagnostics() const { return p_->program_.diagnostics(); } +std::map System::accepted_balance_terms(const std::string& route) const { + if (!p_->external_step_transaction_ || p_->external_step_transaction_committed_) + throw std::runtime_error( + "System::_accepted_balance_terms requires an active uncommitted external step transaction"); + return p_->program_.accepted_balance_terms(route, "System"); +} void System::begin_step_projection_report() { p_->program_.begin_step_projection_report(); } From 142a6f4a7278daeaf2cb0fb59047b4b81dbced0f Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:43:18 +0200 Subject: [PATCH 011/109] test(diagnostics): prove accepted balance evidence --- .../runtime/test_program_context_contract.cpp | 41 +++++++++++++++ .../unit/runtime/test_consumer_authoring.py | 37 ++++++++++++- .../unit/runtime/test_diagnostics_typed.py | 43 +++++++++++++-- .../runtime/test_runtime_instance_gate.py | 40 ++++++++++++++ .../python/unit/time/test_time_ops_polish.py | 52 +++++++++++++++++++ 5 files changed, 209 insertions(+), 4 deletions(-) diff --git a/tests/cpp/unit/runtime/test_program_context_contract.cpp b/tests/cpp/unit/runtime/test_program_context_contract.cpp index b992666a7..dfff041f8 100644 --- a/tests/cpp/unit/runtime/test_program_context_contract.cpp +++ b/tests/cpp/unit/runtime/test_program_context_contract.cpp @@ -124,6 +124,47 @@ TEST(ProgramContextContract, ProjectionReportSurvivesScientificRollbackUntilCons EXPECT_THROW(sim.note_step_projection(""), std::invalid_argument); } +TEST(ProgramContextContract, AcceptedBalanceEvidenceIsCurrentAttemptExactAndFailClosed) { + ensure_kokkos(); + SystemConfig cfg; + cfg.n = 2; + cfg.L = 1.0; + System sim(cfg); + const std::string route = "pops.balance-ledger-route.v1:sha256:" + std::string(64, '1'); + const std::array, 5> terms{{ + {"storage_change", 11.0}, + {"outward_boundary_flux", 2.0}, + {"sources", 5.0}, + {"reflux", 3.0}, + {"projection", 1.0}, + }}; + + sim.begin_step_transaction(); + sim.begin_step_projection_report(); + for (const auto& [name, value] : terms) + sim.record_program_diagnostic("pops.balance-term.v1:" + route + ":" + name, 0.25 * value); + for (const auto& [name, value] : terms) + sim.record_program_diagnostic("pops.balance-term.v1:" + route + ":" + name, 0.75 * value); + const auto accepted = sim.accepted_balance_terms(route); + EXPECT_EQ(accepted.size(), terms.size()); + for (const auto& [name, value] : terms) + EXPECT_DOUBLE_EQ(accepted.at(name), value); + // Reserved balance evidence is deliberately attempt-local and therefore absent + // from the persistent/checkpointed inspection-diagnostic registry. + EXPECT_EQ(sim.program_diagnostics().count("pops.balance-term.v1:" + route + ":storage_change"), + 0u); + sim.rollback_step_transaction(); + EXPECT_THROW((void)sim.accepted_balance_terms(route), std::runtime_error); + + sim.begin_step_transaction(); + sim.begin_step_projection_report(); + for (std::size_t index = 0; index + 1 < terms.size(); ++index) + sim.record_program_diagnostic("pops.balance-term.v1:" + route + ":" + terms[index].first, + terms[index].second); + EXPECT_THROW((void)sim.accepted_balance_terms(route), std::runtime_error); + sim.rollback_step_transaction(); +} + double max_abs_diff(const std::vector& a, const std::vector& b) { double d = 0; for (std::size_t k = 0; k < a.size(); ++k) { diff --git a/tests/python/unit/runtime/test_consumer_authoring.py b/tests/python/unit/runtime/test_consumer_authoring.py index 8bc029f29..8609f7c96 100644 --- a/tests/python/unit/runtime/test_consumer_authoring.py +++ b/tests/python/unit/runtime/test_consumer_authoring.py @@ -3,7 +3,7 @@ import pytest import pops -from pops.diagnostics import Integral, StepChangeNorm +from pops.diagnostics import Balance, BalanceLedger, Integral, StepChangeNorm from pops.domain import Rectangle from pops.frames import Cartesian2D from pops.mesh import LayoutPlanBuilder, normalize_layout_plan @@ -233,6 +233,41 @@ def test_console_monitor_is_a_scheduled_rank_zero_diagnostic_consumer(): ) +def test_balance_consumer_resolves_one_exact_native_ledger_route(): + case, block, state = _case() + clock = Clock("macro", owner=case.owner_path) + schedule = every(4, clock=clock) + ledger = BalanceLedger("mass") + graph = ConsumerGraph.from_consumers(( + ScientificOutput( + format=ParaView(), + schedule=schedule, + fields=(state,), + diagnostics=(Balance(ledger, block=block),), + target="state/balance", + ), + )) + case.consumers(graph) + pops.validate(case) + subjects = case.layout_subjects() + layout = normalize_layout_plan( + Uniform(cartesian_grid(n=8)), + owner=case.owner_path.canonical(), + states=subjects.states, + fields=subjects.fields, + blocks=subjects.blocks, + handle_resolver=case.resolve, + ) + + resolved = graph.resolve(case.resolve, layout, owner=case.owner_path.canonical()) + quantity, = resolved.nodes[0].diagnostic_quantities + operation, = quantity.execution["operations"] + assert operation["reduction"] == "accepted_balance" + assert operation["balance_route"] == ledger.route_identity( + case.resolve(block)).token + assert quantity.reference == case.resolve(state) + + def test_console_monitor_can_be_removed_at_authoring_time(): case, block, _state = _case() monitor = ConsoleMonitor( diff --git a/tests/python/unit/runtime/test_diagnostics_typed.py b/tests/python/unit/runtime/test_diagnostics_typed.py index 289b8dc56..a2670174d 100644 --- a/tests/python/unit/runtime/test_diagnostics_typed.py +++ b/tests/python/unit/runtime/test_diagnostics_typed.py @@ -16,8 +16,8 @@ pops = pytest.importorskip("pops") from pops.descriptors import Descriptor # noqa: E402 -from pops.diagnostics import (ConservationCheck, Integral, MinMax, # noqa: E402 - Norm, StepChangeNorm) +from pops.diagnostics import (Balance, BalanceLedger, ConservationCheck, # noqa: E402 + Integral, MinMax, Norm, StepChangeNorm) from pops.linalg.norms import L1, L2, LInf # noqa: E402 from pops.model import Module # noqa: E402 from pops.physics.roles import Density # noqa: E402 @@ -31,7 +31,10 @@ # --- package surface -------------------------------------------------------------------- def test_typed_measures_exported(): import pops.diagnostics as diag - for name in ("Norm", "Integral", "MinMax", "ConservationCheck", "StepChangeNorm"): + for name in ( + "Balance", "BalanceLedger", "Norm", "Integral", "MinMax", + "ConservationCheck", "StepChangeNorm", + ): assert hasattr(diag, name), name assert name in diag.__all__, name @@ -90,6 +93,39 @@ def test_step_change_norm_is_typed_l2_and_whole_state(): StepChangeNorm("l2") +def test_balance_uses_one_typed_native_attempt_route(): + ledger = BalanceLedger("mass") + balance = Balance(ledger, block=_NE_BLOCK) + execution = balance.diagnostic_execution() + operation, = execution["operations"] + assert operation["name"] == "balance" + assert operation["reduction"] == "accepted_balance" + assert operation["balance_route"].startswith( + "pops.balance-ledger-route.v1:sha256:") + assert execution["role"] is None and execution["conservation"] is None + assert balance.options()["ledger"] == ledger.to_data() + from pops.output._consumer_contracts import diagnostic_collective_operations + + assert diagnostic_collective_operations(execution) == () + mixed = { + **execution, + "operations": execution["operations"] + [{ + "name": "integral", + "reduction": "sum", + "transform": "identity", + "metric_weighted": True, + }], + } + with pytest.raises(ValueError, match="sole diagnostic execution operation"): + diagnostic_collective_operations(mixed) + with pytest.raises(TypeError, match="BalanceLedger"): + Balance("mass", block=_NE_BLOCK) + with pytest.raises(TypeError, match="physics BlockHandle"): + Balance(ledger, block=None) + with pytest.raises(ValueError, match="open-domain Balance"): + ConservationCheck(balance).diagnostic_execution() + + # --- Integral / MinMax ------------------------------------------------------------------ def test_integral_is_a_sum_reduction(): mass = Integral(role=Density()) @@ -204,6 +240,7 @@ def test_conservation_check_rejects_invalid_tolerance_and_multivalued_quantity() # --- inspect() / options() / __repr__ (Spec 5 sec.12.1 printable rule) ------------------ @pytest.mark.parametrize("measure,cls_name,category", [ + (Balance(BalanceLedger("mass"), block=_NE_BLOCK), "Balance", "diagnostic_balance"), (Norm(L2(), block=_NE_BLOCK), "Norm", "diagnostic_norm"), (Integral(role=Density()), "Integral", "diagnostic_integral"), (MinMax(block=_NE_BLOCK), "MinMax", "diagnostic_minmax"), diff --git a/tests/python/unit/runtime/test_runtime_instance_gate.py b/tests/python/unit/runtime/test_runtime_instance_gate.py index c22a9f3be..f2649b8b5 100644 --- a/tests/python/unit/runtime/test_runtime_instance_gate.py +++ b/tests/python/unit/runtime/test_runtime_instance_gate.py @@ -1425,6 +1425,46 @@ def _step_change_l2(self): assert (value, composite) == (0.125, True) +def test_balance_diagnostic_accepts_only_the_exact_native_five_term_tuple(): + from pops.runtime._runtime_consumers import RuntimeConsumerPublisher + + class _Provider: + def _accepted_balance_terms(self, route): + assert route == "pops.balance-ledger-route.v1:sha256:" + "1" * 64 + return { + "storage_change": 11.0, + "outward_boundary_flux": 2.0, + "sources": 5.0, + "reflux": 3.0, + "projection": 1.0, + } + + terms = RuntimeConsumerPublisher._native_balance_terms( + _Provider(), "pops.balance-ledger-route.v1:sha256:" + "1" * 64) + assert terms.residual == 4.0 + assert terms.reflux == 3.0 + + class _Incomplete: + def _accepted_balance_terms(self, _route): + return {"storage_change": 1.0} + + with pytest.raises(TypeError, match="exactly storage_change"): + RuntimeConsumerPublisher._native_balance_terms(_Incomplete(), "route") + + class _Coerced: + def _accepted_balance_terms(self, _route): + return { + "storage_change": "1.0", + "outward_boundary_flux": 2.0, + "sources": 5.0, + "reflux": 3.0, + "projection": 1.0, + } + + with pytest.raises(TypeError, match="exact floating-point"): + RuntimeConsumerPublisher._native_balance_terms(_Coerced(), "route") + + def test_diagnostic_restart_restores_payload_terms_and_native_inspection_registry(): from pops.identity import make_identity from pops.output.data import DiagnosticKey, DiagnosticPayload diff --git a/tests/python/unit/time/test_time_ops_polish.py b/tests/python/unit/time/test_time_ops_polish.py index 76eb7d535..4411299d0 100644 --- a/tests/python/unit/time/test_time_ops_polish.py +++ b/tests/python/unit/time/test_time_ops_polish.py @@ -307,6 +307,58 @@ def test_record_scalar_rejects_non_scalar_and_bad_name(t): raise AssertionError("record_scalar must reject an empty name") +def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): + from pops.diagnostics import BalanceLedger + from pops.diagnostics.balance import BALANCE_TERM_NAMES, balance_record_name + + P = t.Program("p") + U = typed_state(P, "blk") + total = P.sum(U) + ledger = BalanceLedger("mass") + records = P.record_balance( + ledger, + storage_change=total, + outward_boundary_flux=total * 2.0, + sources=total * 3.0, + reflux=total * 0.0, + projection=total * 0.0, + ) + route = ledger.route_identity(U.block) + assert tuple(record.attrs["diagnostic"] for record in records) == tuple( + balance_record_name(route, term) for term in BALANCE_TERM_NAMES) + endpoint = typed_state(P, "blk", state_name="U").next + P.commit(endpoint, P.value("balance_next", U, at=endpoint.point)) + source = emit_cpp_program(P) + assert source.count("ctx.record_scalar(") == 5 + assert route.token in source + + +def test_record_balance_rejects_non_reduced_or_incomplete_evidence(t): + from pops.diagnostics import BalanceLedger + + P = t.Program("p") + U = typed_state(P, "blk") + total = P.sum(U) + with pytest.raises(ValueError, match="global reduction"): + P.record_balance( + BalanceLedger("mass"), + storage_change=P.max_wave_speed(U), + outward_boundary_flux=total, + sources=total, + reflux=total, + projection=total, + ) + with pytest.raises(ValueError, match="additive sum/dot reductions"): + P.record_balance( + BalanceLedger("mass"), + storage_change=P.max(U), + outward_boundary_flux=total, + sources=total, + reflux=total, + projection=total, + ) + + # ---- (A.5) IR hash sensitivity ---- def test_ir_hash_distinguishes_new_ops(t): def _h(build): From c9bab2a2c5fdd2b61db263d9c27caa75f2a1af35 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:43:36 +0200 Subject: [PATCH 012/109] docs(output): document explicit balance ledgers --- docs/design/exact-output-consumers.md | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/docs/design/exact-output-consumers.md b/docs/design/exact-output-consumers.md index 8b81f518b..8a965047c 100644 --- a/docs/design/exact-output-consumers.md +++ b/docs/design/exact-output-consumers.md @@ -425,6 +425,44 @@ quantity an invariant. Diagnostic-only outputs remain valid: their owner-qualifi terms, layout metadata and provenance are preserved even when no field array is selected. Geometry origins and spacings use the conventional `(x, y)` and `(dx, dy)` order. +An executable open-domain balance uses one shared typed identity rather than a Python callback: + +```python +from pops.diagnostics import Balance, BalanceLedger + +mass = BalanceLedger("mass") +program.record_balance( + mass, + storage_change=storage_increment, + outward_boundary_flux=boundary_flux_increment, + sources=source_increment, + reflux=reflux_increment, + projection=projection_increment, +) + +ScientificOutput( + ..., + diagnostics=(Balance(mass, block=fluid),), +) +``` + +Each argument to `record_balance` is a signed, time-integrated native Program sum/dot reduction, +or scalar arithmetic composed only from such reductions and exact literals. +The reported residual is `storage_change + outward_boundary_flux - sources - reflux - projection`. +The native attempt mailbox accumulates repeated cadence/substep invocations, rejects missing or +non-finite terms, and is cleared before the next attempt. The consumer reads it only while the +outer accepted-step transaction still retains the pre-step image. Python therefore packages the +five returned scalars and residual but never traverses arrays, invents a zero term, or reuses a +previous step. A rejected attempt or failed consumer publication restores the mailbox with the +rest of the native transaction. + +This route is explicit evidence, not automatic numerical instrumentation: a Program that cannot +produce its actual reflux or projection increment cannot declare `Balance`. In particular, the +generic automatic extraction of AMR reflux/projection contributions from the internal native +operator ledgers remains separate work. On an adaptive layout the recorded values must already be +composite and coverage-corrected; an ordinary sum of every per-level state would double-count +covered coarse cells. Neither `Balance` nor `BalanceTerms` silently claims otherwise. + Checkpoint remains a separate restart effect. These consumers do not define a checkpoint schema or reader and do not call the scientific-output manifest a restart identity. The checkpoint provider remains the sole owner of sealing, hierarchy/history persistence and strict identity-checked From d20cce86cceb60d38325a9894313b644eb8aa7b2 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:47:30 +0200 Subject: [PATCH 013/109] feat(output): delete ParaView series compatibility route --- python/pops/output/formats.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/python/pops/output/formats.py b/python/pops/output/formats.py index f59f3ecd8..2e7f5b291 100644 --- a/python/pops/output/formats.py +++ b/python/pops/output/formats.py @@ -3,7 +3,6 @@ from dataclasses import dataclass from typing import Any, ClassVar -import warnings from pops.descriptors import Descriptor from pops.descriptors_report import RequirementSet @@ -404,7 +403,6 @@ def __init__( preset: ParaViewPreset | None = None, placement: Any = None, state: Any = _DEFAULT_PARAVIEW_STATE, - series: Any = _UNSET_PARAVIEW_OPTION, ) -> None: selected_mode = _mode( mode, @@ -420,21 +418,6 @@ def __init__( raise ValueError("ParaView.compression must be None or an integer from 0 to 9") if collection is not _UNSET_PARAVIEW_OPTION and type(collection) is not bool: raise TypeError("ParaView.collection must be an exact bool") - if series is not _UNSET_PARAVIEW_OPTION: - if series is not None and type(series) is not bool: - raise TypeError("ParaView.series must be an exact bool or None") - warnings.warn( - "ParaView(series=...) is deprecated; use collection=... for the standard " - "PVD collection", - DeprecationWarning, - stacklevel=2, - ) - legacy_collection = ( - selected_mode is not ParallelMode.PER_RANK if series is None else series) - if collection is not _UNSET_PARAVIEW_OPTION \ - and collection is not legacy_collection: - raise ValueError("ParaView.collection and deprecated series disagree") - collection = legacy_collection if collection is _UNSET_PARAVIEW_OPTION: collection = True from .paraview_state import MaterializedPVSM, PortableState From 4e5a08e505ce09d6f611213365db058cd420cac8 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:47:34 +0200 Subject: [PATCH 014/109] test(output): enforce canonical ParaView collection authoring --- tests/python/unit/output/test_exact_writers.py | 14 +++++--------- .../python/unit/runtime/test_consumer_authoring.py | 2 -- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/tests/python/unit/output/test_exact_writers.py b/tests/python/unit/output/test_exact_writers.py index 7fe6c4ae7..dc3872068 100644 --- a/tests/python/unit/output/test_exact_writers.py +++ b/tests/python/unit/output/test_exact_writers.py @@ -2617,19 +2617,15 @@ def flock(descriptor, operation): def test_generic_series_policy_excludes_paraview_pvd_collections(): + import inspect + assert HDF5().series is True assert NPZ().series is True assert ParaView().series is False assert ParaView().series_catalog() is None - with pytest.warns(DeprecationWarning, match="collection"): - enabled = ParaView(series=True) - assert enabled.collection is True - with pytest.warns(DeprecationWarning, match="collection"): - disabled = ParaView(series=False) - assert disabled.collection is False - with pytest.warns(DeprecationWarning, match="collection"): - with pytest.raises(ValueError, match="disagree"): - ParaView(collection=True, series=False) + assert "series" not in inspect.signature(ParaView).parameters + assert ParaView(collection=True).collection is True + assert ParaView(collection=False).collection is False def test_format_writers_publish_structural_preflight_capabilities(): diff --git a/tests/python/unit/runtime/test_consumer_authoring.py b/tests/python/unit/runtime/test_consumer_authoring.py index 8bc029f29..436be7b20 100644 --- a/tests/python/unit/runtime/test_consumer_authoring.py +++ b/tests/python/unit/runtime/test_consumer_authoring.py @@ -391,8 +391,6 @@ def test_output_format_options_refuse_python_truthiness_coercion() -> None: HDF5(mode="serial") with pytest.raises(TypeError, match="exact bool or None"): HDF5(series=1) - with pytest.raises(TypeError, match="exact bool or None"): - ParaView(series=1) assert HDF5().consumer_data()["options"] == {"mode": "serial", "series": True} serial_options = ParaView().consumer_data()["options"] assert serial_options["mode"] == "serial" From 20e2a7cab3f870784be4b5e7742ac9b81f6ee304 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:47:38 +0200 Subject: [PATCH 015/109] docs(output): record ParaView authoring cutover --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 99374ebd9..e6ce38e46 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning ### Changed +- ParaView output now has one collection-authoring keyword: `collection`. The deprecated + `ParaView(series=...)` compatibility route is deleted instead of being retained beside the + canonical PVD collection contract. - AMR checkpoint capability reports now distinguish same-rank bit-identical replay from non-bit-identical rank-count rematerialization with Dense persisted histories, and state explicitly that `RegridOnRestart()` remains unsupported. The M3 gate now executes the persisted two-rank to From 0f5c076c3d4607f6ac79cdc7a62bed28f05920d4 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:59:51 +0200 Subject: [PATCH 016/109] fix(diagnostics): reserve balance codegen route --- python/pops/codegen/inspect_compiled.py | 2 +- python/pops/codegen/program_emit_kernels.py | 1 + python/pops/codegen/program_emit_ops.py | 13 +++++++++++++ python/pops/time/_history/validation.py | 1 + python/pops/time/_program/authoring.py | 4 ++++ python/pops/time/_program/constants.py | 3 ++- python/pops/time/_program/diagnostics.py | 13 ++++++++++++- python/pops/time/_program/passes.py | 2 +- 8 files changed, 35 insertions(+), 4 deletions(-) diff --git a/python/pops/codegen/inspect_compiled.py b/python/pops/codegen/inspect_compiled.py index 81c8c345d..c98945632 100644 --- a/python/pops/codegen/inspect_compiled.py +++ b/python/pops/codegen/inspect_compiled.py @@ -411,7 +411,7 @@ def _build_arguments( for value in getattr(program, "_values", []): if value.op == "store_history": outputs[value.name or "history"] = {"kind": "history"} - elif value.op == "record" or value.op == "record_scalar": + elif value.op in {"record", "record_scalar", "record_balance_term"}: outputs[value.name or "diagnostic"] = {"kind": "diagnostic"} ghost_depth_by_block = _ghost_depth_by_block(compiled, tuple(instances)) diff --git a/python/pops/codegen/program_emit_kernels.py b/python/pops/codegen/program_emit_kernels.py index 02b85e8cd..1beddc4c2 100644 --- a/python/pops/codegen/program_emit_kernels.py +++ b/python/pops/codegen/program_emit_kernels.py @@ -73,6 +73,7 @@ "fill_boundary", "project", "record_scalar", + "record_balance_term", "cell_compare", "where", "rhs_jacvec", diff --git a/python/pops/codegen/program_emit_ops.py b/python/pops/codegen/program_emit_ops.py index 42c8a5704..018787496 100644 --- a/python/pops/codegen/program_emit_ops.py +++ b/python/pops/codegen/program_emit_ops.py @@ -466,6 +466,19 @@ def _emit_op(program: Any, v: Any, base: Any, committed_ids: Any, var: Any, mode lines.append("ctx.record_scalar(%s, %s);" % (json.dumps(v.attrs["diagnostic"]), var[scalar_in.id])) var[v.id] = var[scalar_in.id] + elif v.op == "record_balance_term": + # Dedicated, non-bindable sink for a validated Program.record_balance term. Ordinary + # record_scalar names cannot enter the reserved native attempt mailbox. + (scalar_in,) = v.inputs + lines.append( + "ctx.record_balance_term(%s, %s, %s);" + % ( + json.dumps(v.attrs["route"]), + json.dumps(v.attrs["term"]), + var[scalar_in.id], + ) + ) + var[v.id] = var[scalar_in.id] elif v.op == "rhs": state_in = v.inputs[0] # rhs inputs = (state[, fields]); the state is first var[v.id] = "r%d" % v.id diff --git a/python/pops/time/_history/validation.py b/python/pops/time/_history/validation.py index bc1e81a55..b54fe1531 100644 --- a/python/pops/time/_history/validation.py +++ b/python/pops/time/_history/validation.py @@ -65,6 +65,7 @@ "hmin", "max_wave_speed", "record_scalar", + "record_balance_term", "reduce", "scalar_op", "compare", diff --git a/python/pops/time/_program/authoring.py b/python/pops/time/_program/authoring.py index d295110cb..0cc1a4df7 100644 --- a/python/pops/time/_program/authoring.py +++ b/python/pops/time/_program/authoring.py @@ -359,6 +359,10 @@ def record_scalar(self, name: Any, value: Any) -> Any: to ``ctx.record_scalar("", )``.""" if not isinstance(name, str) or not name: raise ValueError("record_scalar: name must be a non-empty string") + if name.startswith("pops.balance-term"): + raise ValueError( + "record_scalar: pops.balance-term is reserved for Program.record_balance" + ) if not (isinstance(value, ProgramValue) and value.vtype == "scalar"): raise ValueError("record_scalar: value must be a Scalar value (e.g. P.norm2(R)); got %r" % (value,)) diff --git a/python/pops/time/_program/constants.py b/python/pops/time/_program/constants.py index 767eedd44..dd1999f74 100644 --- a/python/pops/time/_program/constants.py +++ b/python/pops/time/_program/constants.py @@ -41,7 +41,8 @@ class _ProgramConstants: # Deliberately EXCLUDED (kept live): the buffer-writers schur_rhs / schur_explicit_flux / laplacian # / gradient / divergence / apply_laplacian_coeff / schur_coeffs / schur_reconstruct / schur_energy # (alias an input buffer); the side-effecting solve_fields[_from_blocks] / project / fill_boundary / - # store_history / record_scalar; solve_linear (reads its rhs by buffer identity); scalar_field / + # store_history / record_scalar / record_balance_term; solve_linear (reads its rhs by buffer + # identity); scalar_field / # state / history (scratch/state bindings other ops fill or alias); and the sub-block ops below. _REMOVABLE_OPS = frozenset({ "rhs", "source", "apply", "local_transform", "linear_combine", "linear_source", "solve_local_linear", diff --git a/python/pops/time/_program/diagnostics.py b/python/pops/time/_program/diagnostics.py index b2cee7644..bc2158c3d 100644 --- a/python/pops/time/_program/diagnostics.py +++ b/python/pops/time/_program/diagnostics.py @@ -101,7 +101,18 @@ def require_reduced(value: Any, term: str, seen: set[int]) -> ProgramValue: ) route = ledger.route_identity(next(iter(blocks))) return tuple( - self.record_scalar(balance_record_name(route, name), terms[name]) + self._new( + "scalar", + "record_balance_term", + (terms[name],), + { + "diagnostic": balance_record_name(route, name), + "route": route.token, + "term": name, + }, + balance_record_name(route, name), + terms[name].block, + ) for name in BALANCE_TERM_NAMES ) diff --git a/python/pops/time/_program/passes.py b/python/pops/time/_program/passes.py index 9b974ccad..e2fe22fed 100644 --- a/python/pops/time/_program/passes.py +++ b/python/pops/time/_program/passes.py @@ -115,7 +115,7 @@ def eliminate_dead_nodes(self) -> Any: scalar_op, compare) AND no live op consumes its result. EVERY other op -- the buffer-writers that alias a caller-allocated input buffer (schur_rhs, laplacian, gradient, divergence, schur_*), the side-effecting ops (solve_fields, project, fill_boundary, store_history, - record_scalar), solve_linear, and the sub-block-owning ops (while/if/range, + record_scalar, record_balance_term), solve_linear, and the sub-block-owning ops (while/if/range, matrix_free_operator, solve_local_nonlinear) -- is treated as LIVE even when its result looks unconsumed, so an unknown/new op is NEVER wrongly dropped. The live set is reverse-reachability from the commits plus those non-removable nodes. The surviving nodes are renumbered to From f66ea9ce543f57888625012171530069d13191c0 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 22:59:58 +0200 Subject: [PATCH 017/109] fix(runtime): isolate the balance attempt mailbox --- include/pops/runtime/amr_system.hpp | 4 ++ .../runtime/program/amr_program_context.hpp | 4 ++ .../pops/runtime/program/program_context.hpp | 4 ++ .../program/program_execution_services.hpp | 4 ++ .../runtime/program/program_runtime_state.hpp | 64 +++++++++++++------ include/pops/runtime/system.hpp | 4 ++ src/runtime/amr/amr_system.cpp | 4 ++ src/runtime/system/system_program.cpp | 12 ++-- 8 files changed, 76 insertions(+), 24 deletions(-) diff --git a/include/pops/runtime/amr_system.hpp b/include/pops/runtime/amr_system.hpp index ac30577f9..e7e0f9e2c 100644 --- a/include/pops/runtime/amr_system.hpp +++ b/include/pops/runtime/amr_system.hpp @@ -1085,6 +1085,10 @@ class AmrSystem { private: friend class runtime::program::AmrProgramContext; + /// Dedicated generated-Program sink for one validated, attempt-local balance term. It remains + /// private to AmrProgramContext and is deliberately absent from Python bindings. + POPS_EXPORT void record_program_balance_term(const std::string& route, const std::string& term, + double value); /// Read-only compiled-artifact capability check; artifact authority installation is private to /// AmrSystem::install_program and cannot be injected through the public facade. POPS_EXPORT bool program_owns_operator_authority( diff --git a/include/pops/runtime/program/amr_program_context.hpp b/include/pops/runtime/program/amr_program_context.hpp index 9dfb6c231..7ac4055c2 100644 --- a/include/pops/runtime/program/amr_program_context.hpp +++ b/include/pops/runtime/program/amr_program_context.hpp @@ -3687,6 +3687,10 @@ class AmrProgramContext : public ProgramExecutionServices { void program_execution_record_scalar_(const std::string& name, Real value) const { facade_->record_program_diagnostic(name, value); } + void program_execution_record_balance_term_(const std::string& route, const std::string& term, + Real value) const { + facade_->record_program_balance_term(route, term, value); + } void program_execution_note_step_projection_(const std::string& name) const { facade_->note_step_projection(name); } diff --git a/include/pops/runtime/program/program_context.hpp b/include/pops/runtime/program/program_context.hpp index ffe290439..0baa9133e 100644 --- a/include/pops/runtime/program/program_context.hpp +++ b/include/pops/runtime/program/program_context.hpp @@ -1392,6 +1392,10 @@ class ProgramContext : public ProgramExecutionServices { void program_execution_record_scalar_(const std::string& name, Real value) const { sys_->record_program_diagnostic(name, value); } + void program_execution_record_balance_term_(const std::string& route, const std::string& term, + Real value) const { + sys_->record_program_balance_term(route, term, value); + } void program_execution_note_step_projection_(const std::string& name) const { sys_->note_step_projection(name); } diff --git a/include/pops/runtime/program/program_execution_services.hpp b/include/pops/runtime/program/program_execution_services.hpp index 3734c7900..11b38c3fc 100644 --- a/include/pops/runtime/program/program_execution_services.hpp +++ b/include/pops/runtime/program/program_execution_services.hpp @@ -644,6 +644,10 @@ class ProgramExecutionServices { provider_().program_execution_record_scalar_(name, value); } + void record_balance_term(const std::string& route, const std::string& term, Real value) const { + provider_().program_execution_record_balance_term_(route, term, value); + } + void note_step_projection(const std::string& name) const { provider_().program_execution_note_step_projection_(name); } diff --git a/include/pops/runtime/program/program_runtime_state.hpp b/include/pops/runtime/program/program_runtime_state.hpp index 31a74501a..99788de3b 100644 --- a/include/pops/runtime/program/program_runtime_state.hpp +++ b/include/pops/runtime/program/program_runtime_state.hpp @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -678,22 +679,53 @@ struct ProgramRuntimeState { " set_clock cannot reuse an active stride window; restore its strict checkpoint image"); } + static bool has_reserved_balance_namespace(const std::string& name) noexcept { + return name.rfind("pops.balance-term", 0) == 0; + } + + static void require_balance_route(const std::string& route, const std::string& runtime) { + static constexpr std::string_view kRoutePrefix = "pops.balance-ledger-route.v1:sha256:"; + if (route.size() != kRoutePrefix.size() + 64 || + route.compare(0, kRoutePrefix.size(), kRoutePrefix.data(), kRoutePrefix.size()) != 0 || + !std::all_of(route.begin() + static_cast(kRoutePrefix.size()), route.end(), + [](unsigned char value) { + return (value >= '0' && value <= '9') || (value >= 'a' && value <= 'f'); + })) + throw std::invalid_argument(runtime + " requires a canonical balance-ledger-route identity"); + } + + static void require_balance_term(const std::string& term, const std::string& runtime) { + static constexpr std::array kTerms{ + "storage_change", "outward_boundary_flux", "sources", "reflux", "projection"}; + if (std::find(kTerms.begin(), kTerms.end(), std::string_view(term)) == kTerms.end()) + throw std::invalid_argument(runtime + " requires one canonical five-term balance name"); + } + /// Record a compiled-Program scalar. Ordinary P.record_scalar names remain inspectable after the - /// step with last-write-wins semantics. The reserved balance prefix is attempt-local and additive. + /// step with last-write-wins semantics. The balance namespace has a separate typed sink. void record_diagnostic(const std::string& name, Real value) { - // A Program cadence may invoke the compiled body several times inside one public macro-step. - // Balance records are signed, time-integrated increments and therefore accumulate across those - // invocations. Ordinary inspection diagnostics retain their historical last-write-wins contract. - static constexpr const char* kBalancePrefix = "pops.balance-term.v1:"; - if (name.rfind(kBalancePrefix, 0) == 0) { - auto [entry, inserted] = step_balance_terms_.try_emplace(name, value); - if (!inserted) - entry->second += value; - return; - } + if (has_reserved_balance_namespace(name)) + throw std::invalid_argument( + "ProgramRuntimeState::record_diagnostic: pops.balance-term is a reserved namespace"); diagnostics_[name] = value; } + /// Record one validated Program.record_balance term. Not exposed through the Python runtime + /// facade: only generated ProgramContext code reaches this sink. + void record_balance_term(const std::string& route, const std::string& term, Real value, + const std::string& runtime) { + require_balance_route(route, runtime + "::record_balance_term"); + require_balance_term(term, runtime + "::record_balance_term"); + if (!std::isfinite(static_cast(value))) + throw std::invalid_argument(runtime + "::record_balance_term requires a finite value"); + const std::string name = "pops.balance-term.v1:" + route + ":" + term; + // A Program cadence may invoke the compiled body several times inside one public macro-step. + // Terms are signed, time-integrated increments and therefore accumulate across invocations. + auto [entry, inserted] = step_balance_terms_.try_emplace(name, value); + if (!inserted) + entry->second += value; + } + /// Read the named diagnostic, FAIL-LOUD if the Program never recorded it. @p runtime names the /// Program subsystem setter in the message (not a generic getter). @throws std::out_of_range. Real diagnostic(const std::string& name, const std::string& runtime) const { @@ -718,17 +750,9 @@ struct ProgramRuntimeState { /// active. No zero, stale value, or array-derived Python fallback is permitted. std::map accepted_balance_terms(const std::string& route, const std::string& runtime) const { - static constexpr const char* kRoutePrefix = "pops.balance-ledger-route.v1:sha256:"; static constexpr std::array kTerms{"storage_change", "outward_boundary_flux", "sources", "reflux", "projection"}; - const std::string prefix{kRoutePrefix}; - if (route.size() != prefix.size() + 64 || route.compare(0, prefix.size(), prefix) != 0 || - !std::all_of(route.begin() + static_cast(prefix.size()), route.end(), - [](unsigned char value) { - return (value >= '0' && value <= '9') || (value >= 'a' && value <= 'f'); - })) - throw std::invalid_argument( - runtime + "::_accepted_balance_terms requires a canonical balance-ledger-route identity"); + require_balance_route(route, runtime + "::_accepted_balance_terms"); std::map result; for (const char* term : kTerms) { const std::string record = "pops.balance-term.v1:" + route + ":" + term; diff --git a/include/pops/runtime/system.hpp b/include/pops/runtime/system.hpp index c427ee4ae..6a737749b 100644 --- a/include/pops/runtime/system.hpp +++ b/include/pops/runtime/system.hpp @@ -1363,6 +1363,10 @@ class System { private: friend class runtime::program::ProgramContext; friend class PreparedSystemLayoutTransfer; + /// Dedicated generated-Program sink for one validated, attempt-local balance term. It remains + /// private to ProgramContext and is deliberately absent from Python bindings. + POPS_EXPORT void record_program_balance_term(const std::string& route, const std::string& term, + Real value); /// Immediate provider calls are an exported implementation seam for generated ProgramContext /// code, never a public publication route. Every public field solve and every Program solve wraps /// these methods in the same physical accepted/candidate transaction. diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index a9be4b1fa..91eee0341 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -3530,6 +3530,10 @@ pops::runtime::program::Profiler& AmrSystem::profiler_handle() { void AmrSystem::record_program_diagnostic(const std::string& name, double value) { p_->program_.record_diagnostic(name, value); // shared subsystem (ADC-594) } +void AmrSystem::record_program_balance_term(const std::string& route, const std::string& term, + double value) { + p_->program_.record_balance_term(route, term, value, "AmrSystem"); +} double AmrSystem::program_diagnostic(const std::string& name) const { // AMR keeps its historical LENIENT read (missing name -> 0.0), distinct from System's fail-loud // program_diagnostic; not routed through the struct's throwing diagnostic() helper. diff --git a/src/runtime/system/system_program.cpp b/src/runtime/system/system_program.cpp index 43127668f..158fcf5bd 100644 --- a/src/runtime/system/system_program.cpp +++ b/src/runtime/system/system_program.cpp @@ -370,10 +370,10 @@ void System::set_program_block_map(const std::vector& prog_to_sys) { for (std::size_t program = 0; program < prog_to_sys.size(); ++program) { for (std::size_t previous = 0; previous < program; ++previous) { if (prog_to_sys[program] == prog_to_sys[previous]) - throw std::invalid_argument( - "System::set_program_block_map: Program blocks " + std::to_string(previous) + - " and " + std::to_string(program) + " both map to System block " + - std::to_string(prog_to_sys[program])); + throw std::invalid_argument("System::set_program_block_map: Program blocks " + + std::to_string(previous) + " and " + std::to_string(program) + + " both map to System block " + + std::to_string(prog_to_sys[program])); } } p_->program_.block_map_ = prog_to_sys; @@ -411,6 +411,10 @@ void System::block_project(int b, MultiFab& u) { void System::record_program_diagnostic(const std::string& name, Real value) { p_->program_.record_diagnostic(name, value); } +void System::record_program_balance_term(const std::string& route, const std::string& term, + Real value) { + p_->program_.record_balance_term(route, term, value, "System"); +} Real System::program_diagnostic(const std::string& name) const { return p_->program_.diagnostic(name, "System"); } From 126399febb541270f678b670e0b1a91d6d9d9ac5 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 23:00:07 +0200 Subject: [PATCH 018/109] test(diagnostics): reject balance namespace spoofing --- .../runtime/test_program_context_contract.cpp | 23 +++++++++++++++---- .../test_program_execution_services.py | 2 ++ .../python/unit/time/test_time_ops_polish.py | 10 +++++++- 3 files changed, 30 insertions(+), 5 deletions(-) diff --git a/tests/cpp/unit/runtime/test_program_context_contract.cpp b/tests/cpp/unit/runtime/test_program_context_contract.cpp index dfff041f8..d01d02b94 100644 --- a/tests/cpp/unit/runtime/test_program_context_contract.cpp +++ b/tests/cpp/unit/runtime/test_program_context_contract.cpp @@ -130,6 +130,7 @@ TEST(ProgramContextContract, AcceptedBalanceEvidenceIsCurrentAttemptExactAndFail cfg.n = 2; cfg.L = 1.0; System sim(cfg); + ProgramContext context(&sim); const std::string route = "pops.balance-ledger-route.v1:sha256:" + std::string(64, '1'); const std::array, 5> terms{{ {"storage_change", 11.0}, @@ -142,9 +143,9 @@ TEST(ProgramContextContract, AcceptedBalanceEvidenceIsCurrentAttemptExactAndFail sim.begin_step_transaction(); sim.begin_step_projection_report(); for (const auto& [name, value] : terms) - sim.record_program_diagnostic("pops.balance-term.v1:" + route + ":" + name, 0.25 * value); + context.record_balance_term(route, name, 0.25 * value); for (const auto& [name, value] : terms) - sim.record_program_diagnostic("pops.balance-term.v1:" + route + ":" + name, 0.75 * value); + context.record_balance_term(route, name, 0.75 * value); const auto accepted = sim.accepted_balance_terms(route); EXPECT_EQ(accepted.size(), terms.size()); for (const auto& [name, value] : terms) @@ -159,8 +160,22 @@ TEST(ProgramContextContract, AcceptedBalanceEvidenceIsCurrentAttemptExactAndFail sim.begin_step_transaction(); sim.begin_step_projection_report(); for (std::size_t index = 0; index + 1 < terms.size(); ++index) - sim.record_program_diagnostic("pops.balance-term.v1:" + route + ":" + terms[index].first, - terms[index].second); + context.record_balance_term(route, terms[index].first, terms[index].second); + EXPECT_THROW((void)sim.accepted_balance_terms(route), std::runtime_error); + sim.rollback_step_transaction(); + + sim.begin_step_transaction(); + sim.begin_step_projection_report(); + for (const std::string& forged : + {"pops.balance-term", "pops.balance-term.v1", "pops.balance-term.v1:forged"}) { + EXPECT_THROW(sim.record_program_diagnostic(forged, 1.0), std::invalid_argument); + EXPECT_EQ(sim.program_diagnostics().count(forged), 0u); + } + EXPECT_THROW((void)sim.accepted_balance_terms(route), std::runtime_error); + EXPECT_THROW( + context.record_balance_term("pops.balance-ledger-route.v1:sha256:bad", "storage_change", 1.0), + std::invalid_argument); + EXPECT_THROW(context.record_balance_term(route, "unknown", 1.0), std::invalid_argument); EXPECT_THROW((void)sim.accepted_balance_terms(route), std::runtime_error); sim.rollback_step_transaction(); } diff --git a/tests/python/architecture/test_program_execution_services.py b/tests/python/architecture/test_program_execution_services.py index ddf66e182..25e1ee5bc 100644 --- a/tests/python/architecture/test_program_execution_services.py +++ b/tests/python/architecture/test_program_execution_services.py @@ -74,6 +74,7 @@ "int n_blocks(", "Real physical_time(", "void record_scalar(", + "void record_balance_term(", "RuntimeParams program_params(", "void set_field_logical_timepoint(", "void set_field_boundary_parameters(", @@ -233,6 +234,7 @@ def test_contexts_expose_explicit_provider_hooks_for_the_shared_surface(): "program_execution_block_count_", "program_execution_physical_time_", "program_execution_record_scalar_", + "program_execution_record_balance_term_", "program_execution_params_", "program_execution_set_field_timepoint_", "program_execution_set_field_parameters_", diff --git a/tests/python/unit/time/test_time_ops_polish.py b/tests/python/unit/time/test_time_ops_polish.py index 4411299d0..fb6218a00 100644 --- a/tests/python/unit/time/test_time_ops_polish.py +++ b/tests/python/unit/time/test_time_ops_polish.py @@ -329,7 +329,8 @@ def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): endpoint = typed_state(P, "blk", state_name="U").next P.commit(endpoint, P.value("balance_next", U, at=endpoint.point)) source = emit_cpp_program(P) - assert source.count("ctx.record_scalar(") == 5 + assert source.count("ctx.record_balance_term(") == 5 + assert "ctx.record_scalar(" not in source assert route.token in source @@ -339,6 +340,13 @@ def test_record_balance_rejects_non_reduced_or_incomplete_evidence(t): P = t.Program("p") U = typed_state(P, "blk") total = P.sum(U) + for forged in ( + "pops.balance-term", + "pops.balance-term.v1", + "pops.balance-term.v1:forged", + ): + with pytest.raises(ValueError, match="reserved for Program.record_balance"): + P.record_scalar(forged, total) with pytest.raises(ValueError, match="global reduction"): P.record_balance( BalanceLedger("mass"), From c9429df5431cafb0f1e3498e282eea70dec80aea Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 23:00:14 +0200 Subject: [PATCH 019/109] docs(output): reserve the balance term namespace --- docs/design/exact-output-consumers.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/design/exact-output-consumers.md b/docs/design/exact-output-consumers.md index 8a965047c..8d2ff4c1a 100644 --- a/docs/design/exact-output-consumers.md +++ b/docs/design/exact-output-consumers.md @@ -456,6 +456,10 @@ five returned scalars and residual but never traverses arrays, invents a zero te previous step. A rejected attempt or failed consumer publication restores the mailbox with the rest of the native transaction. +The `pops.balance-term` namespace is reserved. Ordinary `Program.record_scalar(...)` authoring and +the Python runtime diagnostic binding both reject it; generated `record_balance` code reaches a +separate native sink that validates the route and canonical term before touching the mailbox. + This route is explicit evidence, not automatic numerical instrumentation: a Program that cannot produce its actual reflux or projection increment cannot declare `Balance`. In particular, the generic automatic extraction of AMR reflux/projection contributions from the internal native From 6874ccb6873795a40751d00b98f717c8f9374c5e Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 23:01:10 +0200 Subject: [PATCH 020/109] test(diagnostics): keep the balance sink private --- .../architecture/test_program_execution_services.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/python/architecture/test_program_execution_services.py b/tests/python/architecture/test_program_execution_services.py index 25e1ee5bc..6dba3f6d7 100644 --- a/tests/python/architecture/test_program_execution_services.py +++ b/tests/python/architecture/test_program_execution_services.py @@ -9,6 +9,10 @@ SHARED = PROGRAM_DIR / "program_execution_services.hpp" UNIFORM = PROGRAM_DIR / "program_context.hpp" AMR = PROGRAM_DIR / "amr_program_context.hpp" +BINDINGS = ( + ROOT / "python" / "bindings" / "core" / "init" / "init_system.cpp", + ROOT / "python" / "bindings" / "core" / "init" / "init_amr.cpp", +) CODEGEN = ROOT / "python" / "pops" / "codegen" CODEGEN_CONTEXT_ROUTES = ( CODEGEN / "program_codegen.py", @@ -131,6 +135,11 @@ def test_uniform_and_amr_inherit_the_same_execution_service(): ) +def test_balance_attempt_sink_is_not_python_bound(): + for binding in BINDINGS: + assert "record_program_balance_term" not in _read(binding) + + def test_codegen_uses_one_facade_selected_provider_factory_not_concrete_context_dispatch(): shared = _read(SHARED) uniform = _read(UNIFORM) From 6410c18f2782f7e88ec27e34e42997fabf04ad9a Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 23:08:27 +0200 Subject: [PATCH 021/109] docs(output): state balance cadence cost --- docs/design/exact-output-consumers.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/design/exact-output-consumers.md b/docs/design/exact-output-consumers.md index 8d2ff4c1a..fc4304481 100644 --- a/docs/design/exact-output-consumers.md +++ b/docs/design/exact-output-consumers.md @@ -460,6 +460,15 @@ The `pops.balance-term` namespace is reserved. Ordinary `Program.record_scalar(. the Python runtime diagnostic binding both reject it; generated `record_balance` code reaches a separate native sink that validates the route and canonical term before touching the mailbox. +`record_balance` is not currently gated by the matching Consumer cadence. Its five term-producing +Program reduction paths run whenever execution reaches the call, including every cadence/substep, +even if the `Balance` consumer is due only every N accepted steps. Use this explicit route with a +dense (every-invocation) balance cadence unless that collective cost is intentionally acceptable: +a sparse Consumer cadence does not save the upstream reductions. Scheduling only the five terminal +record nodes would not fix this, because their reduction inputs would still execute. A future +low-overhead sparse route therefore needs one typed due decision shared by the Program and +ConsumerGraph. + This route is explicit evidence, not automatic numerical instrumentation: a Program that cannot produce its actual reflux or projection increment cannot declare `Balance`. In particular, the generic automatic extraction of AMR reflux/projection contributions from the internal native From 36bbbf30f6cf8651529e96130fea831977eeb70e Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 23:14:14 +0200 Subject: [PATCH 022/109] test(mpi): run layout transfers on execution communicator --- .../mpi/test_mpi_system_layout_transfer.cpp | 44 ++++++++++++++++--- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/tests/cpp/integration/mpi/test_mpi_system_layout_transfer.cpp b/tests/cpp/integration/mpi/test_mpi_system_layout_transfer.cpp index 14ab2bfae..ca0243838 100644 --- a/tests/cpp/integration/mpi/test_mpi_system_layout_transfer.cpp +++ b/tests/cpp/integration/mpi/test_mpi_system_layout_transfer.cpp @@ -179,9 +179,34 @@ pops::SystemLayoutTransferSpec transfer_spec() { POPS_TRANSFER_OPERATION_CONSERVATIVE_CELL_AVERAGE_V1}; } -pops::SystemLayoutTransferExecution transfer_execution() { +class ScopedMpiCommunicator { + public: + explicit ScopedMpiCommunicator(MPI_Comm source) { + if (MPI_Comm_dup(source, &communicator_) != MPI_SUCCESS) + throw std::runtime_error("MPI_Comm_dup failed for the layout-transfer test lane"); + if (MPI_Comm_set_errhandler(communicator_, MPI_ERRORS_RETURN) != MPI_SUCCESS) { + MPI_Comm_free(&communicator_); + throw std::runtime_error("MPI_Comm_set_errhandler failed for the layout-transfer test lane"); + } + } + + ~ScopedMpiCommunicator() { + if (communicator_ != MPI_COMM_NULL) + MPI_Comm_free(&communicator_); + } + + ScopedMpiCommunicator(const ScopedMpiCommunicator&) = delete; + ScopedMpiCommunicator& operator=(const ScopedMpiCommunicator&) = delete; + + MPI_Comm get() const { return communicator_; } + + private: + MPI_Comm communicator_ = MPI_COMM_NULL; +}; + +pops::SystemLayoutTransferExecution transfer_execution(MPI_Comm communicator) { return {1, - "test::execution::mpi-world-host", + "test::execution::mpi-lane-host", POPS_MEMORY_SPACE_HOST_V1, "test::backend::mpi-cpu", "test::device::cpu:0", @@ -192,9 +217,9 @@ pops::SystemLayoutTransferExecution transfer_execution() { POPS_PRECISION_FLOAT64_V1, 0, "test::stream::host-synchronous", - static_cast(MPI_Comm_c2f(MPI_COMM_WORLD)), + static_cast(MPI_Comm_c2f(communicator)), static_cast(MPI_Type_c2f(MPI_DOUBLE)), - "MPI_COMM_WORLD", + "test::mpi-system-layout-transfer-lane", "MPI_DOUBLE"}; } @@ -289,6 +314,13 @@ int run_mpi_system_layout_transfer(int argc, char** argv) { return finish(); { + const ScopedMpiCommunicator transfer_lane(MPI_COMM_WORLD); + int world_relation = MPI_UNEQUAL; + check(MPI_Comm_compare(transfer_lane.get(), MPI_COMM_WORLD, &world_relation) == MPI_SUCCESS, + "layout-transfer lane comparison succeeds"); + check(world_relation == MPI_CONGRUENT, + "layout-transfer test executes on a distinct world-congruent communicator"); + std::shared_ptr component; bool healthy = phase("authenticated Transfer DSO load", [&] { component = std::make_shared( @@ -335,7 +367,7 @@ int run_mpi_system_layout_transfer(int argc, char** argv) { "coarse System has one owner and one empty peer"); healthy = phase("collective prepared Transfer construction", [&] { transfer = pops::PreparedSystemLayoutTransfer::prepare( - *fine, *coarse, component, transfer_spec(), transfer_execution()); + *fine, *coarse, component, transfer_spec(), transfer_execution(transfer_lane.get())); }); } @@ -348,7 +380,7 @@ int run_mpi_system_layout_transfer(int argc, char** argv) { receipt.source_layout_identity == kFineLayout && receipt.target_layout_identity == kCoarseLayout && receipt.source_block == "fine" && receipt.target_block == "coarse" && - receipt.execution_identity == "test::execution::mpi-world-host" && + receipt.execution_identity == "test::execution::mpi-lane-host" && receipt.operation == POPS_TRANSFER_OPERATION_CONSERVATIVE_CELL_AVERAGE_V1 && receipt.generation == generation && receipt.attempt == attempt && receipt.source_element_count == 16 && receipt.destination_element_count == 4, From fae33d5c301ddb56c847cb4822052e7a28510deb Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 23:16:07 +0200 Subject: [PATCH 023/109] fix(output): refuse start-time balance schedules --- python/pops/output/_consumer_contracts.py | 10 +++++++ python/pops/time/_schedule/api.py | 34 +++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/python/pops/output/_consumer_contracts.py b/python/pops/output/_consumer_contracts.py index 3c8d7bc58..046896b35 100644 --- a/python/pops/output/_consumer_contracts.py +++ b/python/pops/output/_consumer_contracts.py @@ -576,6 +576,16 @@ def __post_init__(self) -> None: ConsumerKind.DIAGNOSTIC, ConsumerKind.SCIENTIFIC_OUTPUT}: raise ValueError( "only ConsoleMonitor or ScientificOutput can carry diagnostic quantities") + has_accepted_balance = any( + operation["reduction"] == "accepted_balance" + for quantity in diagnostic_quantities + for operation in quantity.execution["operations"] + ) + if has_accepted_balance and self.schedule.consumer_may_fire_at_start(): + raise ValueError( + "Balance schedule cannot fire at_start: accepted balance evidence exists " + "only after a native step attempt" + ) object.__setattr__(self, "diagnostic_quantities", diagnostic_quantities) if not isinstance(self.dependencies, tuple): raise TypeError("ConsumerManifest.dependencies must be a tuple") diff --git a/python/pops/time/_schedule/api.py b/python/pops/time/_schedule/api.py index ea9aaea52..2e1820421 100644 --- a/python/pops/time/_schedule/api.py +++ b/python/pops/time/_schedule/api.py @@ -109,6 +109,14 @@ def consumer_due(self, coordinate: int, moment: Any) -> bool: "schedule trigger %s does not implement consumer_due()" % type(self).__name__ ) + def consumer_may_fire_at_start(self) -> bool: + """Whether this trigger can publish before the first accepted step. + + Unknown extension triggers are conservatively start-capable until they override this + planning capability. This lets accepted-step-only consumers fail closed at bind time. + """ + return True + def consumer_next_deadline(self, *, physical_time_hex: str) -> str | None: """Return the next hard physical-time boundary, if this trigger owns one. @@ -145,6 +153,9 @@ def consumer_due(self, coordinate: int, moment: Any) -> bool: del coordinate return not moment.at_start + def consumer_may_fire_at_start(self) -> bool: + return False + @stable_component_identity("pops://time/schedule/triggers/every") @dataclass(frozen=True, slots=True) @@ -169,6 +180,9 @@ def schedule_params(self) -> dict[str, Any]: def consumer_due(self, coordinate: int, moment: Any) -> bool: return not moment.at_start and coordinate % self.n == 0 + def consumer_may_fire_at_start(self) -> bool: + return False + def _canonical_binary64(value: Any, *, where: str, positive: bool = False) -> float: if isinstance(value, bool) or not isinstance(value, (int, float)): @@ -273,6 +287,9 @@ def consumer_due(self, coordinate: int, moment: Any) -> bool: target = _every_dt_lattice_time(nearest, self.interval) return math.isfinite(target) and now >= target and _same_physical_time(now, target) + def consumer_may_fire_at_start(self) -> bool: + return False + def consumer_occurrence_evidence( self, coordinate: int, moment: Any, ) -> dict[str, Any] | None: @@ -324,6 +341,9 @@ def consumer_due(self, coordinate: int, moment: Any) -> bool: del coordinate return moment.at_start + def consumer_may_fire_at_start(self) -> bool: + return True + @stable_component_identity("pops://time/schedule/triggers/at-end") @dataclass(frozen=True, slots=True) @@ -338,6 +358,9 @@ def consumer_due(self, coordinate: int, moment: Any) -> bool: del coordinate return not moment.at_start and moment.at_end + def consumer_may_fire_at_start(self) -> bool: + return False + @stable_component_identity("pops://time/schedule/triggers/when") @dataclass(frozen=True, slots=True) @@ -360,6 +383,9 @@ def consumer_due(self, coordinate: int, moment: Any) -> bool: raise UnresolvedScheduleCondition(self.condition) return self.condition + def consumer_may_fire_at_start(self) -> bool: + return False + @stable_component_identity("pops://time/schedule/off-policy") @dataclass(frozen=True, slots=True) @@ -518,6 +544,14 @@ def is_always(self) -> bool: raise TypeError("Trigger.is_always() must return an exact bool") return result + def consumer_may_fire_at_start(self) -> bool: + result = self.trigger.consumer_may_fire_at_start() + if type(result) is not bool: + raise TypeError( + "Trigger.consumer_may_fire_at_start() must return an exact bool" + ) + return result + def needs_cache(self) -> bool: if self.off is None: return False From 51fa332530e78e793899fbc45e8ac6f0ec9756e6 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 23:16:38 +0200 Subject: [PATCH 024/109] test(output): prove balance starts after an attempt --- .../unit/runtime/test_consumer_authoring.py | 39 ++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/tests/python/unit/runtime/test_consumer_authoring.py b/tests/python/unit/runtime/test_consumer_authoring.py index 8609f7c96..4fe90c4e8 100644 --- a/tests/python/unit/runtime/test_consumer_authoring.py +++ b/tests/python/unit/runtime/test_consumer_authoring.py @@ -25,7 +25,7 @@ from pops.output._consumer_contracts import ConsumerKind, ParallelMode from pops.representations import Conservative from pops.spaces import CellState -from pops.time import Clock, FailRun as SolveFailRun, every +from pops.time import Clock, FailRun as SolveFailRun, every, on_start from tests.python.support.layout_plan import cartesian_grid @@ -262,12 +262,49 @@ def test_balance_consumer_resolves_one_exact_native_ledger_route(): resolved = graph.resolve(case.resolve, layout, owner=case.owner_path.canonical()) quantity, = resolved.nodes[0].diagnostic_quantities operation, = quantity.execution["operations"] + assert not schedule.consumer_may_fire_at_start() assert operation["reduction"] == "accepted_balance" assert operation["balance_route"] == ledger.route_identity( case.resolve(block)).token assert quantity.reference == case.resolve(state) +def test_balance_consumer_refuses_a_schedule_that_can_fire_at_start(): + case, block, state = _case() + clock = Clock("macro", owner=case.owner_path) + schedule = on_start(clock=clock) + graph = ConsumerGraph.from_consumers(( + ScientificOutput( + format=ParaView(), + schedule=schedule, + fields=(state,), + diagnostics=(Balance(BalanceLedger("mass"), block=block),), + target="state/balance", + ), + )) + case.consumers(graph) + pops.validate(case) + subjects = case.layout_subjects() + layout = normalize_layout_plan( + Uniform(cartesian_grid(n=8)), + owner=case.owner_path.canonical(), + states=subjects.states, + fields=subjects.fields, + blocks=subjects.blocks, + handle_resolver=case.resolve, + ) + + assert schedule.consumer_may_fire_at_start() + with pytest.raises( + ValueError, + match=( + "Balance schedule cannot fire at_start: accepted balance evidence " + "exists only after a native step attempt" + ), + ): + graph.resolve(case.resolve, layout, owner=case.owner_path.canonical()) + + def test_console_monitor_can_be_removed_at_authoring_time(): case, block, _state = _case() monitor = ConsoleMonitor( From 5e19b9a972e5b7a6323596b595bf33908d295c47 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Wed, 29 Jul 2026 23:59:25 +0200 Subject: [PATCH 025/109] feat(output): fuse accepted balance cadence --- include/pops/runtime/amr_system.hpp | 2 + .../runtime/program/amr_program_context.hpp | 4 + .../pops/runtime/program/program_context.hpp | 4 + .../program/program_execution_services.hpp | 5 + .../runtime/program/program_runtime_state.hpp | 58 ++++ include/pops/runtime/system.hpp | 2 + .../runtime/system/system_program_driver.hpp | 38 +-- python/pops/codegen/_compile_drivers.py | 12 +- python/pops/codegen/_phases.py | 8 +- python/pops/codegen/program_balance_due.py | 259 ++++++++++++++++++ python/pops/codegen/program_codegen.py | 14 +- python/pops/codegen/program_emit_control.py | 14 +- python/pops/codegen/program_emit_ops.py | 55 ++-- python/pops/codegen/program_graph_lowering.py | 3 +- python/pops/output/_balance_due_contract.py | 202 ++++++++++++++ src/runtime/amr/amr_system.cpp | 43 +-- src/runtime/system/system_program.cpp | 4 + 17 files changed, 668 insertions(+), 59 deletions(-) create mode 100644 python/pops/codegen/program_balance_due.py create mode 100644 python/pops/output/_balance_due_contract.py diff --git a/include/pops/runtime/amr_system.hpp b/include/pops/runtime/amr_system.hpp index e7e0f9e2c..c6dd89255 100644 --- a/include/pops/runtime/amr_system.hpp +++ b/include/pops/runtime/amr_system.hpp @@ -1089,6 +1089,8 @@ class AmrSystem { /// private to AmrProgramContext and is deliberately absent from Python bindings. POPS_EXPORT void record_program_balance_term(const std::string& route, const std::string& term, double value); + POPS_EXPORT bool program_balance_consumer_is_due(const std::string& contract, + const std::string& route, int every_n) const; /// Read-only compiled-artifact capability check; artifact authority installation is private to /// AmrSystem::install_program and cannot be injected through the public facade. POPS_EXPORT bool program_owns_operator_authority( diff --git a/include/pops/runtime/program/amr_program_context.hpp b/include/pops/runtime/program/amr_program_context.hpp index 7ac4055c2..4ea773439 100644 --- a/include/pops/runtime/program/amr_program_context.hpp +++ b/include/pops/runtime/program/amr_program_context.hpp @@ -3691,6 +3691,10 @@ class AmrProgramContext : public ProgramExecutionServices { Real value) const { facade_->record_program_balance_term(route, term, value); } + bool program_execution_balance_consumer_is_due_(const std::string& contract, + const std::string& route, int every_n) const { + return facade_->program_balance_consumer_is_due(contract, route, every_n); + } void program_execution_note_step_projection_(const std::string& name) const { facade_->note_step_projection(name); } diff --git a/include/pops/runtime/program/program_context.hpp b/include/pops/runtime/program/program_context.hpp index 0baa9133e..8a7601545 100644 --- a/include/pops/runtime/program/program_context.hpp +++ b/include/pops/runtime/program/program_context.hpp @@ -1396,6 +1396,10 @@ class ProgramContext : public ProgramExecutionServices { Real value) const { sys_->record_program_balance_term(route, term, value); } + bool program_execution_balance_consumer_is_due_(const std::string& contract, + const std::string& route, int every_n) const { + return sys_->program_balance_consumer_is_due(contract, route, every_n); + } void program_execution_note_step_projection_(const std::string& name) const { sys_->note_step_projection(name); } diff --git a/include/pops/runtime/program/program_execution_services.hpp b/include/pops/runtime/program/program_execution_services.hpp index 11b38c3fc..0dc2963e9 100644 --- a/include/pops/runtime/program/program_execution_services.hpp +++ b/include/pops/runtime/program/program_execution_services.hpp @@ -552,6 +552,11 @@ class ProgramExecutionServices { return profiler().schedule_decision(due, cache_backed); } + bool balance_consumer_is_due(const std::string& contract, const std::string& route, + int every_n) const { + return provider_().program_execution_balance_consumer_is_due_(contract, route, every_n); + } + /// Scheduler cache semantics shared by every capable Program storage provider. /// /// The service owns cadence, profiling and value movement. A provider supplies only the diff --git a/include/pops/runtime/program/program_runtime_state.hpp b/include/pops/runtime/program/program_runtime_state.hpp index 99788de3b..815205379 100644 --- a/include/pops/runtime/program/program_runtime_state.hpp +++ b/include/pops/runtime/program/program_runtime_state.hpp @@ -264,6 +264,11 @@ struct ProgramRuntimeState { /// consumers read it while the facade's outer transaction still retains U^n, so a missing term /// cannot silently reuse the preceding step. std::map step_balance_terms_; + /// Attempt-local outer accepted-step target used by ConsumerGraph-fused balance guards. Program + /// substeps temporarily publish their window-start macro step through the facade, so generated + /// balance code must not infer the public target from `macro_step()+1`. + bool balance_due_window_active_ = false; + int balance_due_target_step_ = 0; /// Attempt-local identities of ProjectAndRecheck branches that actually executed. This report /// mailbox is cleared at attempt entry and consumed by the Python transaction coordinator before /// commit or rollback; it is deliberately not checkpoint or accepted scientific state. @@ -694,6 +699,19 @@ struct ProgramRuntimeState { throw std::invalid_argument(runtime + " requires a canonical balance-ledger-route identity"); } + static void require_balance_due_contract(const std::string& contract, + const std::string& runtime) { + static constexpr std::string_view kContractPrefix = "pops.balance-due-contract.v1:sha256:"; + if (contract.size() != kContractPrefix.size() + 64 || + contract.compare(0, kContractPrefix.size(), kContractPrefix.data(), + kContractPrefix.size()) != 0 || + !std::all_of(contract.begin() + static_cast(kContractPrefix.size()), + contract.end(), [](unsigned char value) { + return (value >= '0' && value <= '9') || (value >= 'a' && value <= 'f'); + })) + throw std::invalid_argument(runtime + " requires a canonical balance-due-contract identity"); + } + static void require_balance_term(const std::string& term, const std::string& runtime) { static constexpr std::array kTerms{ "storage_change", "outward_boundary_flux", "sources", "reflux", "projection"}; @@ -743,6 +761,8 @@ struct ProgramRuntimeState { void begin_step_projection_report() { step_projections_.clear(); step_balance_terms_.clear(); + balance_due_window_active_ = false; + balance_due_target_step_ = 0; } /// Return exactly the five native Program scalars recorded for one typed balance route during the @@ -771,6 +791,44 @@ struct ProgramRuntimeState { return result; } + void begin_balance_due_window(int accepted_macro_step, const std::string& runtime) { + if (balance_due_window_active_) + throw std::logic_error(runtime + " balance due window is already active"); + if (accepted_macro_step < 0 || accepted_macro_step == std::numeric_limits::max()) + throw std::overflow_error(runtime + " balance due target step is not representable"); + balance_due_target_step_ = accepted_macro_step + 1; + balance_due_window_active_ = true; + } + + void end_balance_due_window() noexcept { + balance_due_window_active_ = false; + balance_due_target_step_ = 0; + } + + template + void run_balance_due_window(int accepted_macro_step, const std::string& runtime, Body&& body) { + begin_balance_due_window(accepted_macro_step, runtime); + try { + std::forward(body)(); + } catch (...) { + end_balance_due_window(); + throw; + } + end_balance_due_window(); + } + + bool balance_consumer_is_due(const std::string& contract, const std::string& route, int every_n, + const std::string& runtime) const { + require_balance_due_contract(contract, runtime + "::balance_consumer_is_due"); + require_balance_route(route, runtime + "::balance_consumer_is_due"); + if (every_n <= 0) + throw std::invalid_argument(runtime + "::balance_consumer_is_due requires a positive period"); + if (!balance_due_window_active_ || balance_due_target_step_ <= 0) + throw std::logic_error(runtime + + "::balance_consumer_is_due requires an active public-step window"); + return balance_due_target_step_ % every_n == 0; + } + void note_step_projection(const std::string& name) { if (name.empty()) throw std::invalid_argument("Program step projection identity cannot be empty"); diff --git a/include/pops/runtime/system.hpp b/include/pops/runtime/system.hpp index 8c65f9686..50a937da5 100644 --- a/include/pops/runtime/system.hpp +++ b/include/pops/runtime/system.hpp @@ -1367,6 +1367,8 @@ class System { /// private to ProgramContext and is deliberately absent from Python bindings. POPS_EXPORT void record_program_balance_term(const std::string& route, const std::string& term, Real value); + POPS_EXPORT bool program_balance_consumer_is_due(const std::string& contract, + const std::string& route, int every_n) const; /// Immediate provider calls are an exported implementation seam for generated ProgramContext /// code, never a public publication route. Every public field solve and every Program solve wraps /// these methods in the same physical accepted/candidate transaction. diff --git a/include/pops/runtime/system/system_program_driver.hpp b/include/pops/runtime/system/system_program_driver.hpp index 29ab321a1..ad9cdb7da 100644 --- a/include/pops/runtime/system/system_program_driver.hpp +++ b/include/pops/runtime/system/system_program_driver.hpp @@ -181,24 +181,26 @@ class SystemProgramDriver { throw std::logic_error("System Program cadence window starts before macro-step zero"); const int window_start_macro_step = accepted_macro_step - held_before_due; try { - for (int sub = 0; sub < n; ++sub) { - const auto partition = P->program_.prepare_cadence_substep(cadence, sub, n, "System"); - // Publish the exact accepted start of this Program substep. ProgramContext derives every - // stage/boundary physical coordinate from System::time(); leaving the facade at the outer - // macro-step start would stamp every substep with the same time and would start a stride - // catch-up window one held step too late. - P->t = partition.start; - // A due stride is one logical public window, irrespective of the number of internal - // substeps. Publish its accepted start tick for every Program invocation; schedules and - // contexts must not mistake internal calls for additional public macro-steps. - P->macro_step_ = window_start_macro_step; - // Record the dt handed to the program BEFORE the call so the runtime's store_history can tag - // the slot it produces with the exact dt (ADC-626 variable-dt replay). Shared by step() and - // step_cfl() (both route here), so no call site is missed. A plain data assignment. - P->program_.last_dt_ = static_cast(partition.dt); - P->program_.step_(partition.dt); - P->t = partition.end; - } + P->program_.run_balance_due_window(accepted_macro_step, "System", [&] { + for (int sub = 0; sub < n; ++sub) { + const auto partition = P->program_.prepare_cadence_substep(cadence, sub, n, "System"); + // Publish the exact accepted start of this Program substep. ProgramContext derives every + // stage/boundary physical coordinate from System::time(); leaving the facade at the outer + // macro-step start would stamp every substep with the same time and would start a stride + // catch-up window one held step too late. + P->t = partition.start; + // A due stride is one logical public window, irrespective of the number of internal + // substeps. Publish its accepted start tick for every Program invocation; schedules and + // contexts must not mistake internal calls for additional public macro-steps. + P->macro_step_ = window_start_macro_step; + // Record the dt handed to the program BEFORE the call so the runtime's store_history can + // tag the slot it produces with the exact dt (ADC-626 variable-dt replay). Shared by + // step() and step_cfl() (both route here), so no call site is missed. + P->program_.last_dt_ = static_cast(partition.dt); + P->program_.step_(partition.dt); + P->t = partition.end; + } + }); } catch (...) { P->t = accepted_time; P->macro_step_ = accepted_macro_step; diff --git a/python/pops/codegen/_compile_drivers.py b/python/pops/codegen/_compile_drivers.py index 7347160e8..01ddb5997 100644 --- a/python/pops/codegen/_compile_drivers.py +++ b/python/pops/codegen/_compile_drivers.py @@ -186,7 +186,7 @@ def compile_problem(so_path: Any = None, *, model: Any = None, model_graph: Any backend: Any = "production", target: Any = "system", force: Any = False, cxx: Any = None, include: Any = None, std: Any = None, debug: Any = False, libraries: Any = None, problem_snapshot: Any = None, - field_plans: Any = None) -> Any: + field_plans: Any = None, balance_due_contract: Any = None) -> Any: """Compile a time Program into an ABI-compatible native ``problem.so``. Only the production backend is supported; ``target`` selects system or AMR entrypoints. An @@ -234,12 +234,20 @@ def compile_problem(so_path: Any = None, *, model: Any = None, model_graph: Any from pops.time._program.detach import detach_compiled_program time = detach_compiled_program(time) program_graph = time.to_graph() + from pops.output._balance_due_contract import BalanceDueContract + if balance_due_contract is None: + balance_due_contract = BalanceDueContract.from_consumer_graph(None) + if type(balance_due_contract) is not BalanceDueContract: + raise TypeError( + "compile_problem balance_due_contract must be an exact BalanceDueContract" + ) from pops.codegen.program_emit_kernels import _prepared_native_components native_components = _prepared_native_components(time) from pops.codegen.program_graph_lowering import emit_program_graph src = emit_program_graph( program_graph, lowering_program=time, model=model, - model_graph=model_graph, target=target, field_plans=field_plans) + model_graph=model_graph, target=target, field_plans=field_plans, + balance_due_contract=balance_due_contract) include = include or pops_include() sig = pops_header_signature(include) diff --git a/python/pops/codegen/_phases.py b/python/pops/codegen/_phases.py index ed3c18187..9c1365344 100644 --- a/python/pops/codegen/_phases.py +++ b/python/pops/codegen/_phases.py @@ -335,15 +335,20 @@ def compile(plan: Any) -> Any: from pops.codegen._compile_drivers import compile_problem from pops.codegen._compiled_artifact import CompiledLayoutProgram from pops.codegen.program_models import ProgramModelGraph + from pops.codegen.program_balance_due import validate_balance_due_contract + from pops.output._balance_due_contract import BalanceDueContract program = None options = dict(plan.compile_options) options["libraries"] = plan.libraries + balance_due_contract = BalanceDueContract.from_consumer_graph(plan.consumer_graph) + validate_balance_due_contract(plan.time, balance_due_contract) if len(plan.layout_plan.layouts) == 1: model_graph = build_program_model_graph(plan) program = compile_problem( time=plan.time, model_graph=model_graph, backend=plan.backend, target=plan.target, - problem_snapshot=plan.snapshot, field_plans=plan.field_plans, **options) + problem_snapshot=plan.snapshot, field_plans=plan.field_plans, + balance_due_contract=balance_due_contract, **options) program._discard_authoring() row = plan.layout_plan.layouts[0] layout_programs = (CompiledLayoutProgram( @@ -378,6 +383,7 @@ def compile(plan: Any) -> Any: target=plan.layout_targets[layout_id], problem_snapshot=plan.snapshot, field_plans={}, + balance_due_contract=balance_due_contract, **slice_options, ) compiled_program._discard_authoring() diff --git a/python/pops/codegen/program_balance_due.py b/python/pops/codegen/program_balance_due.py new file mode 100644 index 000000000..0d4923f07 --- /dev/null +++ b/python/pops/codegen/program_balance_due.py @@ -0,0 +1,259 @@ +"""Compile-time fusion of accepted Balance consumers into Program scalar producers.""" +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +import json +from types import MappingProxyType +from typing import Any + +from pops.diagnostics.balance import BALANCE_TERM_NAMES +from pops.identity import Identity +from pops.output._balance_due_contract import BalanceDueContract +from pops.time.values import ProgramValue + + +@dataclass(frozen=True, slots=True) +class BalanceDueLowering: + """Immutable lowering facts for one Program and exact ConsumerGraph contract.""" + + contract: Identity + route_periods: Mapping[str, tuple[int, ...]] + guarded_values: Mapping[int, tuple[str, ...]] + record_routes: Mapping[int, str] + + def __post_init__(self) -> None: + if ( + type(self.contract) is not Identity + or self.contract.domain != "balance-due-contract" + or self.contract.schema_version != 1 + ): + raise TypeError( + "BalanceDueLowering.contract must be a version-1 balance-due-contract Identity" + ) + object.__setattr__( + self, + "route_periods", + MappingProxyType(dict(self.route_periods)), + ) + object.__setattr__( + self, + "guarded_values", + MappingProxyType(dict(self.guarded_values)), + ) + object.__setattr__( + self, + "record_routes", + MappingProxyType(dict(self.record_routes)), + ) + + +def _attribute_sources(value: ProgramValue) -> tuple[ProgramValue, ...]: + sources = [] + for key in ( + "true_result", + "false_result", + "body", + "residual", + "apply_result", + ): + candidate = value.attrs.get(key) + if isinstance(candidate, ProgramValue): + sources.append(candidate) + return tuple(sources) + + +def _program_balance_records( + program: Any, +) -> tuple[ + tuple[ProgramValue, ...], + dict[int, str], + dict[str, dict[str, ProgramValue]], +]: + from pops.codegen.program_lowerability import all_ops + + operations = tuple(all_ops(program)) + ids = [value.id for value in operations] + if len(ids) != len(set(ids)): + raise ValueError("Program balance due lowering requires globally unique SSA ids") + record_routes: dict[int, str] = {} + terms: dict[str, dict[str, ProgramValue]] = {} + for value in operations: + if value.op != "record_balance_term": + continue + route = Identity.from_token(value.attrs.get("route")) + if ( + route.domain != "balance-ledger-route" + or route.schema_version != 1 + or value.attrs.get("term") not in BALANCE_TERM_NAMES + ): + raise ValueError( + "record_balance_term requires one canonical route and five-term name" + ) + term = value.attrs["term"] + by_term = terms.setdefault(route.token, {}) + if term in by_term: + raise ValueError( + "Program records balance route %s term %s more than once" + % (route.token, term) + ) + by_term[term] = value + record_routes[value.id] = route.token + expected = set(BALANCE_TERM_NAMES) + for route, by_term in terms.items(): + if set(by_term) != expected: + missing = sorted(expected.difference(by_term)) + extra = sorted(set(by_term).difference(expected)) + raise ValueError( + "Program balance route %s must record exactly five terms; missing=%s extra=%s" + % (route, missing, extra) + ) + return operations, record_routes, terms + + +def validate_balance_due_contract(program: Any, contract: Any) -> None: + """Fail before codegen when a Balance consumer has no matching five-term producer.""" + if type(contract) is not BalanceDueContract: + raise TypeError( + "balance due validation requires an exact BalanceDueContract" + ) + _operations, _records, terms = _program_balance_records(program) + missing = sorted( + row.route.token for row in contract.routes if row.route.token not in terms + ) + if missing: + raise ValueError( + "ConsumerGraph Balance routes have no Program.record_balance producer: %s" + % ", ".join(missing) + ) + + +def prepare_balance_due_lowering( + program: Any, + contract: Any, +) -> BalanceDueLowering: + """Return exclusive balance-producer guards without mutating the Program graph.""" + if type(contract) is not BalanceDueContract: + raise TypeError( + "balance due lowering requires an exact BalanceDueContract" + ) + operations, record_routes, terms = _program_balance_records(program) + route_periods = { + route: ( + () if (row := contract.route(route)) is None + else row.accepted_step_periods() + ) + for route in terms + } + by_id = {value.id: value for value in operations} + required_routes: dict[int, set[str]] = {} + + def require(value: ProgramValue, route: str) -> None: + if value.op not in {"reduce", "scalar_op"}: + raise ValueError( + "record_balance producer %r is not an additive reduction/scalar chain" + % value.name + ) + routes = required_routes.setdefault(value.id, set()) + if route in routes: + return + routes.add(route) + if value.op == "scalar_op": + for source in value.inputs: + require(source, route) + + for record_id, route in record_routes.items(): + record = by_id[record_id] + if len(record.inputs) != 1: + raise ValueError("record_balance_term must consume one exact scalar") + require(record.inputs[0], route) + + balance_nodes = set(required_routes).union(record_routes) + consumers: dict[int, set[int]] = {value_id: set() for value_id in by_id} + for consumer in operations: + for source in (*consumer.inputs, *_attribute_sources(consumer)): + consumers.setdefault(source.id, set()).add(consumer.id) + + # A scalar chain shared with a non-balance use remains unconditional. Propagate that liveness + # backwards so an upstream reduction cannot be skipped while a downstream ordinary diagnostic + # still reads it. + always_required = { + value_id + for value_id in balance_nodes + if any(consumer not in balance_nodes for consumer in consumers.get(value_id, ())) + } + pending = list(always_required) + while pending: + value = by_id[pending.pop()] + for source in value.inputs: + if source.id in balance_nodes and source.id not in always_required: + always_required.add(source.id) + pending.append(source.id) + + guarded = { + value_id: tuple(sorted(routes)) + for value_id, routes in required_routes.items() + if value_id not in always_required + } + return BalanceDueLowering( + contract.identity, + route_periods, + guarded, + record_routes, + ) + + +def emit_balance_due_guards( + lowering: BalanceDueLowering, + var: dict[Any, Any], + lines: list[str], +) -> None: + """Emit one host-side due decision per recorded route before any balance collective.""" + if type(lowering) is not BalanceDueLowering: + raise TypeError("balance due guard emission requires BalanceDueLowering") + contract = json.dumps(lowering.contract.token) + for index, (route, periods) in enumerate(sorted(lowering.route_periods.items())): + if not periods: + token = "false" + else: + calls = [ + "ctx.balance_consumer_is_due(%s, %s, %d)" + % (contract, json.dumps(route), period) + for period in periods + ] + token = "balance_due_%d" % index + lines.append("const bool %s = (%s);" % (token, " || ".join(calls))) + var[("balance_due_route", route)] = token + var[("balance_guarded_values",)] = lowering.guarded_values + var[("balance_record_routes",)] = lowering.record_routes + + +def balance_value_due_expression(var: Mapping[Any, Any], value_id: int) -> str | None: + routes = var.get(("balance_guarded_values",), {}).get(value_id) + if routes is None: + return None + tokens = tuple(var[("balance_due_route", route)] for route in routes) + if "true" in tokens: + return "true" + tokens = tuple(token for token in tokens if token != "false") + return "false" if not tokens else "(" + " || ".join(tokens) + ")" + + +def balance_record_due_expression(var: Mapping[Any, Any], value_id: int) -> str: + route = var.get(("balance_record_routes",), {}).get(value_id) + if not isinstance(route, str) or not route: + raise ValueError("record_balance_term lost its compile-time due route") + token = var.get(("balance_due_route", route)) + if not isinstance(token, str) or not token: + raise ValueError("record_balance_term route has no compile-time due decision") + return token + + +__all__ = [ + "BalanceDueLowering", + "balance_record_due_expression", + "balance_value_due_expression", + "emit_balance_due_guards", + "prepare_balance_due_lowering", + "validate_balance_due_contract", +] diff --git a/python/pops/codegen/program_codegen.py b/python/pops/codegen/program_codegen.py index 9851836c5..3cca937e1 100644 --- a/python/pops/codegen/program_codegen.py +++ b/python/pops/codegen/program_codegen.py @@ -117,6 +117,7 @@ def emit_cpp_program( *, model_graph: Any = None, field_plans: Any = None, + balance_due_contract: Any = None, ) -> str: """Generate the C++ source of a problem.so implementing this Program (codegen). @@ -198,10 +199,21 @@ def emit_cpp_program( authority = model_graph if model_graph is not None else model if target not in ("system", "amr_system"): raise ValueError("emit_cpp_program: target 'system' | 'amr_system' (got %r)" % (target,)) + from pops.output._balance_due_contract import BalanceDueContract + if balance_due_contract is None: + balance_due_contract = BalanceDueContract.from_consumer_graph(None) + if type(balance_due_contract) is not BalanceDueContract: + raise TypeError( + "emit_cpp_program balance_due_contract must be an exact BalanceDueContract" + ) program.validate() _check_lowerable(program, authority, field_plans or {}, target=target) prelude, body, operator_authorities = _emit_body( - program, authority, target=target, field_plans=field_plans or {} + program, + authority, + target=target, + field_plans=field_plans or {}, + balance_due_contract=balance_due_contract, ) # Optional dt bound (spec s18 / ADC-417): emit the SECOND ABI pair -- pops_program_has_dt_bound() # (true iff a bound was set) and one target-qualified entry accepting the authenticated runtime diff --git a/python/pops/codegen/program_emit_control.py b/python/pops/codegen/program_emit_control.py index ff7397cfd..30e0ee0b8 100644 --- a/python/pops/codegen/program_emit_control.py +++ b/python/pops/codegen/program_emit_control.py @@ -171,7 +171,7 @@ def _emit_contiguous_rhs_group( def _emit_body(program: Any, model: Any = None, target: Any = "system", - field_plans: Any = None) -> tuple: + field_plans: Any = None, balance_due_contract: Any = None) -> tuple: """Generate the C++ of the install function in TWO phases (each list indented uniformly by the template). Assumes `_check_lowerable` has passed. @p model supplies the symbolic coefficients of the Phase-4b source / apply / solve_local_linear ops. Returns ``(prelude, body)``: @@ -243,6 +243,18 @@ def _emit_body(program: Any, model: Any = None, target: Any = "system", -1 if owner_index is None else int(owner_index), json.dumps(state_identity), json.dumps(space_identity), json.dumps(row["clock"]), json.dumps(interpolation))) + from pops.codegen.program_balance_due import ( + emit_balance_due_guards, + prepare_balance_due_lowering, + ) + if balance_due_contract is None: + from pops.output._balance_due_contract import BalanceDueContract + balance_due_contract = BalanceDueContract.from_consumer_graph(None) + emit_balance_due_guards( + prepare_balance_due_lowering(program, balance_due_contract), + var, + lines, + ) values = list(program._values) index = 0 # Group identities occupy compiler-reserved slots after the authored SSA namespace. They are diff --git a/python/pops/codegen/program_emit_ops.py b/python/pops/codegen/program_emit_ops.py index aa0fd8f0e..b26afb95c 100644 --- a/python/pops/codegen/program_emit_ops.py +++ b/python/pops/codegen/program_emit_ops.py @@ -555,15 +555,20 @@ def _emit_op(program: Any, v: Any, base: Any, committed_ids: Any, var: Any, mode elif v.op == "record_balance_term": # Dedicated, non-bindable sink for a validated Program.record_balance term. Ordinary # record_scalar names cannot enter the reserved native attempt mailbox. + from pops.codegen.program_balance_due import balance_record_due_expression + (scalar_in,) = v.inputs - lines.append( - "ctx.record_balance_term(%s, %s, %s);" - % ( - json.dumps(v.attrs["route"]), - json.dumps(v.attrs["term"]), - var[scalar_in.id], + due = balance_record_due_expression(var, v.id) + if due != "false": + lines.append( + "if (%s) { ctx.record_balance_term(%s, %s, %s); }" + % ( + due, + json.dumps(v.attrs["route"]), + json.dumps(v.attrs["term"]), + var[scalar_in.id], + ) ) - ) var[v.id] = var[scalar_in.id] elif v.op == "rhs": state_in = v.inputs[0] # rhs inputs = (state[, fields]); the state is first @@ -787,12 +792,10 @@ def _emit_op(program: Any, v: Any, base: Any, committed_ids: Any, var: Any, mode owner = _required_block_index(block_idx, v.block, "reduce value %r" % v.name) if kind == "norm2": (u,) = v.inputs - lines.append("const pops::Real %s = ctx.norm2(%d, %s);" - % (var[v.id], owner, var[u.id])) + reduction = "ctx.norm2(%d, %s)" % (owner, var[u.id]) elif kind == "norm_inf": (u,) = v.inputs - lines.append("const pops::Real %s = ctx.norm_inf(%d, %s);" - % (var[v.id], owner, var[u.id])) + reduction = "ctx.norm_inf(%d, %s)" % (owner, var[u.id]) elif kind in ("sum", "max", "min", "abs_sum"): (u,) = v.inputs comp = int(v.attrs.get("comp", 0)) @@ -802,12 +805,25 @@ def _emit_op(program: Any, v: Any, base: Any, committed_ids: Any, var: Any, mode "min": "min_component", "abs_sum": "abs_sum_component", }[kind] - lines.append("const pops::Real %s = ctx.%s(%d, %s, %d);" - % (var[v.id], context_op, owner, var[u.id], comp)) + reduction = "ctx.%s(%d, %s, %d)" % ( + context_op, + owner, + var[u.id], + comp, + ) else: # dot a, b = v.inputs - lines.append("const pops::Real %s = ctx.dot(%d, %s, %s);" - % (var[v.id], owner, var[a.id], var[b.id])) + reduction = "ctx.dot(%d, %s, %s)" % ( + owner, + var[a.id], + var[b.id], + ) + from pops.codegen.program_balance_due import balance_value_due_expression + + due = balance_value_due_expression(var, v.id) + if due is not None: + reduction = "(%s) ? (%s) : pops::Real(0)" % (due, reduction) + lines.append("const pops::Real %s = %s;" % (var[v.id], reduction)) elif v.op == "cfl": # The dt_bound's runtime cfl argument -- the C++ parameter of pops_program_dt_bound. It is # NOT a statement; its token is the bound parameter name (spec s18 / ADC-417). @@ -836,8 +852,13 @@ def _emit_op(program: Any, v: Any, base: Any, committed_ids: Any, var: Any, mode else: # a literal constant toks.append(scalar_cpp(val)) cppop = {"add": "+", "sub": "-", "mul": "*", "div": "/"}[v.attrs["fn"]] - lines.append("const pops::Real %s = (%s %s %s);" - % (var[v.id], toks[0], cppop, toks[1])) + expression = "(%s %s %s)" % (toks[0], cppop, toks[1]) + from pops.codegen.program_balance_due import balance_value_due_expression + + due = balance_value_due_expression(var, v.id) + if due is not None: + expression = "(%s) ? (%s) : pops::Real(0)" % (due, expression) + lines.append("const pops::Real %s = %s;" % (var[v.id], expression)) elif v.op == "compare": # A predicate over scalars -> an inline boolean C++ expression (no statement of its own; the # while op embeds it directly in `if (!()) break;`). diff --git a/python/pops/codegen/program_graph_lowering.py b/python/pops/codegen/program_graph_lowering.py index fff69526c..407b11c91 100644 --- a/python/pops/codegen/program_graph_lowering.py +++ b/python/pops/codegen/program_graph_lowering.py @@ -7,6 +7,7 @@ def emit_program_graph( graph: Any, *, lowering_program: Any, model: Any = None, model_graph: Any = None, target: str = "system", field_plans: Any = None, + balance_due_contract: Any = None, ) -> str: """Lower exactly ``graph`` through its frozen, graph-equivalent Program adapter.""" from pops.time import ProgramGraph @@ -21,7 +22,7 @@ def emit_program_graph( source = emit_cpp_program( lowering_program, model=model, model_graph=model_graph, target=target, - field_plans=field_plans, + field_plans=field_plans, balance_due_contract=balance_due_contract, ) if lowering_program.to_graph().graph_hash != graph.graph_hash: raise RuntimeError("ProgramGraph lowering mutated or diverged from its compiler input") diff --git a/python/pops/output/_balance_due_contract.py b/python/pops/output/_balance_due_contract.py new file mode 100644 index 000000000..b19711fc3 --- /dev/null +++ b/python/pops/output/_balance_due_contract.py @@ -0,0 +1,202 @@ +"""Typed compile-time bridge from one resolved ConsumerGraph to Balance producers.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from pops.identity import Identity, make_identity +from pops.time._schedule.api import Always, Every, Schedule, When +from pops.time._schedule.domains import AcceptedStep + + +def _identity(value: Any, domain: str, *, where: str) -> Identity: + if type(value) is not Identity or value.domain != domain or value.schema_version != 1: + raise TypeError("%s must be an exact version-1 %s Identity" % (where, domain)) + return Identity.from_data(value.to_data()) + + +@dataclass(frozen=True, slots=True) +class BalanceDueConsumer: + """One exact ConsumerGraph node whose schedule requests a balance route.""" + + consumer: Identity + schedule: Schedule + + def __post_init__(self) -> None: + object.__setattr__( + self, + "consumer", + _identity( + self.consumer, + "consumer-manifest", + where="BalanceDueConsumer.consumer", + ), + ) + if type(self.schedule) is not Schedule: + raise TypeError("BalanceDueConsumer.schedule must be an exact Schedule") + + def to_data(self) -> dict[str, Any]: + return { + "consumer": self.consumer.to_data(), + "schedule": self.schedule.to_data(), + } + + +@dataclass(frozen=True, slots=True) +class BalanceDueRoute: + """All consumer schedules that request one exact native balance route.""" + + route: Identity + consumers: tuple[BalanceDueConsumer, ...] + + def __post_init__(self) -> None: + object.__setattr__( + self, + "route", + _identity( + self.route, + "balance-ledger-route", + where="BalanceDueRoute.route", + ), + ) + if not isinstance(self.consumers, tuple) or any( + type(value) is not BalanceDueConsumer for value in self.consumers + ): + raise TypeError( + "BalanceDueRoute.consumers must contain exact BalanceDueConsumer values" + ) + consumers = tuple( + sorted(self.consumers, key=lambda value: value.consumer.token) + ) + identities = [value.consumer.token for value in consumers] + if len(identities) != len(set(identities)): + raise ValueError("BalanceDueRoute contains a duplicate consumer") + object.__setattr__(self, "consumers", consumers) + + def to_data(self) -> dict[str, Any]: + return { + "route": self.route.to_data(), + "consumers": [value.to_data() for value in self.consumers], + } + + def accepted_step_periods(self) -> tuple[int, ...]: + """Return exact native periods, conservatively using period one when unprovable. + + ``Every(n)`` on the accepted-step domain is the first optimized cutover. ``Always`` and a + statically true ``When`` are exactly period one; a statically false ``When`` contributes no + occurrence. Any other domain/trigger remains active every step so this optimization can + never suppress evidence required by a ConsumerGraph extension or physical-time cadence. + """ + periods = [] + for row in self.consumers: + schedule = row.schedule + if type(schedule.domain) is not AcceptedStep: + return (1,) + trigger = schedule.trigger + if type(trigger) is Every: + periods.append(trigger.n) + elif type(trigger) is Always: + periods.append(1) + elif type(trigger) is When and type(trigger.condition) is bool: + if trigger.condition: + periods.append(1) + else: + return (1,) + if 1 in periods: + return (1,) + return tuple(sorted(set(periods))) + + +@dataclass(frozen=True, slots=True) +class BalanceDueContract: + """Immutable ConsumerGraph-derived cadence authority consumed by native codegen.""" + + consumer_graph: Identity | None + routes: tuple[BalanceDueRoute, ...] + identity: Identity = field(init=False) + + def __post_init__(self) -> None: + if self.consumer_graph is not None: + object.__setattr__( + self, + "consumer_graph", + _identity( + self.consumer_graph, + "consumer-graph", + where="BalanceDueContract.consumer_graph", + ), + ) + if not isinstance(self.routes, tuple) or any( + type(value) is not BalanceDueRoute for value in self.routes + ): + raise TypeError( + "BalanceDueContract.routes must contain exact BalanceDueRoute values" + ) + routes = tuple(sorted(self.routes, key=lambda value: value.route.token)) + tokens = [value.route.token for value in routes] + if len(tokens) != len(set(tokens)): + raise ValueError("BalanceDueContract contains a duplicate route") + object.__setattr__(self, "routes", routes) + object.__setattr__( + self, + "identity", + make_identity("balance-due-contract", self._payload()), + ) + + @classmethod + def from_consumer_graph(cls, graph: Any) -> BalanceDueContract: + from pops.output._consumer_contracts import ConsumerGraph + + if graph is None: + return cls(None, ()) + if type(graph) is not ConsumerGraph or not graph.is_resolved: + raise TypeError( + "BalanceDueContract requires an exact resolved ConsumerGraph or None" + ) + by_route: dict[str, tuple[Identity, list[BalanceDueConsumer]]] = {} + for manifest in graph.nodes: + for quantity in manifest.diagnostic_quantities: + for operation in quantity.execution["operations"]: + if operation["reduction"] != "accepted_balance": + continue + route = Identity.from_token(operation["balance_route"]) + _identity( + route, + "balance-ledger-route", + where="accepted balance operation route", + ) + existing = by_route.setdefault(route.token, (route, [])) + existing[1].append( + BalanceDueConsumer(manifest.identity, manifest.schedule) + ) + return cls( + graph.identity, + tuple( + BalanceDueRoute(route, tuple(consumers)) + for route, consumers in by_route.values() + ), + ) + + def _payload(self) -> dict[str, Any]: + return { + "schema_version": 1, + "consumer_graph": ( + None if self.consumer_graph is None else self.consumer_graph.to_data() + ), + "routes": [value.to_data() for value in self.routes], + } + + def to_data(self) -> dict[str, Any]: + return {**self._payload(), "identity": self.identity.to_data()} + + def route(self, route: str) -> BalanceDueRoute | None: + if not isinstance(route, str) or not route: + raise TypeError("balance due route lookup requires non-empty text") + return next((value for value in self.routes if value.route.token == route), None) + + +__all__ = [ + "BalanceDueConsumer", + "BalanceDueContract", + "BalanceDueRoute", +] diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index 91eee0341..9eefff360 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -409,24 +409,27 @@ struct AmrSystem::Impl { throw std::logic_error("AmrSystem Program cadence window starts before macro-step zero"); const int window_start_macro_step = accepted_macro_step - held_before_due; try { - for (int s = 0; s < program_.substeps_; ++s) { - const auto partition = - program_.prepare_cadence_substep(cadence, s, program_.substeps_, "AmrSystem"); - // AmrProgramContext reads the facade clock at Program entry. Move it to the exact accepted - // start of this substep so stage/tagger coordinates cover the whole catch-up window instead - // of repeating the outer macro-step time. - t = partition.start; - // All internal calls belong to one public stride window. Publish the accepted start tick - // so schedules, regridding and AmrProgramContext never count Program substeps as facade - // macro-steps. - macro_step_ = window_start_macro_step; - // ADC-626/ADC-631: expose this interval before the Program stores its pre-commit history - // sample. The ring ledger then records the outgoing dt from that sample toward the next - // accepted sample (variable-dt replay). Parity with SystemProgramDriver::run_program_cadence. - program_.last_dt_ = static_cast(partition.dt); - program_.step_(partition.dt); - t = partition.end; - } + program_.run_balance_due_window(accepted_macro_step, "AmrSystem", [&] { + for (int s = 0; s < program_.substeps_; ++s) { + const auto partition = + program_.prepare_cadence_substep(cadence, s, program_.substeps_, "AmrSystem"); + // AmrProgramContext reads the facade clock at Program entry. Move it to the exact + // accepted start of this substep so stage/tagger coordinates cover the whole catch-up + // window instead of repeating the outer macro-step time. + t = partition.start; + // All internal calls belong to one public stride window. Publish the accepted start tick + // so schedules, regridding and AmrProgramContext never count Program substeps as facade + // macro-steps. + macro_step_ = window_start_macro_step; + // ADC-626/ADC-631: expose this interval before the Program stores its pre-commit history + // sample. The ring ledger then records the outgoing dt from that sample toward the next + // accepted sample (variable-dt replay). Parity with + // SystemProgramDriver::run_program_cadence. + program_.last_dt_ = static_cast(partition.dt); + program_.step_(partition.dt); + t = partition.end; + } + }); } catch (...) { t = accepted_time; macro_step_ = accepted_macro_step; @@ -3534,6 +3537,10 @@ void AmrSystem::record_program_balance_term(const std::string& route, const std: double value) { p_->program_.record_balance_term(route, term, value, "AmrSystem"); } +bool AmrSystem::program_balance_consumer_is_due(const std::string& contract, + const std::string& route, int every_n) const { + return p_->program_.balance_consumer_is_due(contract, route, every_n, "AmrSystem"); +} double AmrSystem::program_diagnostic(const std::string& name) const { // AMR keeps its historical LENIENT read (missing name -> 0.0), distinct from System's fail-loud // program_diagnostic; not routed through the struct's throwing diagnostic() helper. diff --git a/src/runtime/system/system_program.cpp b/src/runtime/system/system_program.cpp index 158fcf5bd..1231e80ad 100644 --- a/src/runtime/system/system_program.cpp +++ b/src/runtime/system/system_program.cpp @@ -415,6 +415,10 @@ void System::record_program_balance_term(const std::string& route, const std::st Real value) { p_->program_.record_balance_term(route, term, value, "System"); } +bool System::program_balance_consumer_is_due(const std::string& contract, const std::string& route, + int every_n) const { + return p_->program_.balance_consumer_is_due(contract, route, every_n, "System"); +} Real System::program_diagnostic(const std::string& name) const { return p_->program_.diagnostic(name, "System"); } From cb65ace650e8ccbfd039759381446a967018794a Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 00:02:52 +0200 Subject: [PATCH 026/109] test(output): prove sparse balance reductions --- .../runtime/test_program_runtime.cpp | 22 +++ .../test_program_execution_services.py | 2 + .../unit/runtime/test_consumer_authoring.py | 9 +- .../python/unit/time/test_time_ops_polish.py | 158 +++++++++++++++++- 4 files changed, 187 insertions(+), 4 deletions(-) diff --git a/tests/cpp/integration/runtime/test_program_runtime.cpp b/tests/cpp/integration/runtime/test_program_runtime.cpp index 069923faf..122aa7940 100644 --- a/tests/cpp/integration/runtime/test_program_runtime.cpp +++ b/tests/cpp/integration/runtime/test_program_runtime.cpp @@ -117,6 +117,28 @@ static void add_diffusive_gas(System& system, double gamma) { add_compiled_model(system, "gas", model, "none", "rusanov", "conservative", "explicit", gamma); } +TEST(ProgramRuntime, BalanceDueWindowUsesTheOuterAcceptedStepAndCleansUpOnFailure) { + runtime::program::ProgramRuntimeState state; + const std::string contract = "pops.balance-due-contract.v1:sha256:" + std::string(64, '1'); + const std::string route = "pops.balance-ledger-route.v1:sha256:" + std::string(64, '2'); + + EXPECT_THROW((void)state.balance_consumer_is_due(contract, route, 3, "test"), std::logic_error); + state.run_balance_due_window(2, "test", [&] { + EXPECT_TRUE(state.balance_consumer_is_due(contract, route, 3, "test")); + EXPECT_FALSE(state.balance_consumer_is_due(contract, route, 2, "test")); + EXPECT_THROW((void)state.balance_consumer_is_due(contract, route, 0, "test"), + std::invalid_argument); + EXPECT_THROW((void)state.balance_consumer_is_due("forged", route, 3, "test"), + std::invalid_argument); + }); + EXPECT_THROW((void)state.balance_consumer_is_due(contract, route, 3, "test"), std::logic_error); + + EXPECT_THROW( + state.run_balance_due_window(3, "test", [] { throw std::runtime_error("attempt rejected"); }), + std::runtime_error); + EXPECT_THROW((void)state.balance_consumer_is_due(contract, route, 4, "test"), std::logic_error); +} + TEST(ProgramRuntime, ReplayAuthorityRequiresAnArtifactAndAnExactRingDepthPair) { runtime::program::ProgramRuntimeState state; state.history_replay_authorities_ = {{"gas.previous", 3}}; diff --git a/tests/python/architecture/test_program_execution_services.py b/tests/python/architecture/test_program_execution_services.py index 6dba3f6d7..03df11d0d 100644 --- a/tests/python/architecture/test_program_execution_services.py +++ b/tests/python/architecture/test_program_execution_services.py @@ -79,6 +79,7 @@ "Real physical_time(", "void record_scalar(", "void record_balance_term(", + "bool balance_consumer_is_due(", "RuntimeParams program_params(", "void set_field_logical_timepoint(", "void set_field_boundary_parameters(", @@ -244,6 +245,7 @@ def test_contexts_expose_explicit_provider_hooks_for_the_shared_surface(): "program_execution_physical_time_", "program_execution_record_scalar_", "program_execution_record_balance_term_", + "program_execution_balance_consumer_is_due_", "program_execution_params_", "program_execution_set_field_timepoint_", "program_execution_set_field_parameters_", diff --git a/tests/python/unit/runtime/test_consumer_authoring.py b/tests/python/unit/runtime/test_consumer_authoring.py index 4fe90c4e8..d5852394d 100644 --- a/tests/python/unit/runtime/test_consumer_authoring.py +++ b/tests/python/unit/runtime/test_consumer_authoring.py @@ -23,6 +23,7 @@ ) from pops.linalg.norms import L2 from pops.output._consumer_contracts import ConsumerKind, ParallelMode +from pops.output._balance_due_contract import BalanceDueContract from pops.representations import Conservative from pops.spaces import CellState from pops.time import Clock, FailRun as SolveFailRun, every, on_start @@ -262,11 +263,15 @@ def test_balance_consumer_resolves_one_exact_native_ledger_route(): resolved = graph.resolve(case.resolve, layout, owner=case.owner_path.canonical()) quantity, = resolved.nodes[0].diagnostic_quantities operation, = quantity.execution["operations"] + contract = BalanceDueContract.from_consumer_graph(resolved) + route = ledger.route_identity(case.resolve(block)) assert not schedule.consumer_may_fire_at_start() assert operation["reduction"] == "accepted_balance" - assert operation["balance_route"] == ledger.route_identity( - case.resolve(block)).token + assert operation["balance_route"] == route.token assert quantity.reference == case.resolve(state) + assert contract.consumer_graph == resolved.identity + assert contract.route(route.token).accepted_step_periods() == (4,) + assert contract.identity.domain == "balance-due-contract" def test_balance_consumer_refuses_a_schedule_that_can_fire_at_start(): diff --git a/tests/python/unit/time/test_time_ops_polish.py b/tests/python/unit/time/test_time_ops_polish.py index d7cbe065c..089ee956c 100644 --- a/tests/python/unit/time/test_time_ops_polish.py +++ b/tests/python/unit/time/test_time_ops_polish.py @@ -33,13 +33,19 @@ from pops.codegen.program_codegen import emit_cpp_program from pops.domain import Rectangle from pops.frames import Cartesian2D +from pops.identity import make_identity from pops.layouts import Uniform from pops.math import ddt, div, sqrt from pops.mesh import CartesianGrid, PeriodicAxes from pops.numerics import DiscretizationPlan, reconstruction, riemann, variables from pops.numerics.spatial import FiniteVolume from pops.numerics.terms import DefaultSource, Flux -from pops.time import FixedDt +from pops.output._balance_due_contract import ( + BalanceDueConsumer, + BalanceDueContract, + BalanceDueRoute, +) +from pops.time import FixedDt, every, every_dt, when from typed_program_support import typed_state @@ -53,6 +59,24 @@ def t(): return time +def _balance_due_contract(route, *schedules): + return BalanceDueContract( + make_identity("consumer-graph", {"test": "balance-due"}), + ( + BalanceDueRoute( + route, + tuple( + BalanceDueConsumer( + make_identity("consumer-manifest", {"index": index}), + schedule, + ) + for index, schedule in enumerate(schedules) + ), + ), + ), + ) + + # ---- (A.1) solve_local_nonlinear (op 10): the per-cell Newton builder (ADC-422) ---- def test_solve_local_nonlinear_validates_inputs(t): from pops.solvers.nonlinear import LocalNewton @@ -328,12 +352,142 @@ def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): balance_record_name(route, term) for term in BALANCE_TERM_NAMES) endpoint = typed_state(P, "blk", state_name="U").next P.commit(endpoint, P.value("balance_next", U, at=endpoint.point)) - source = emit_cpp_program(P) + contract = _balance_due_contract(route, every(3, clock=P.clock)) + source = emit_cpp_program(P, balance_due_contract=contract) assert source.count("ctx.record_balance_term(") == 5 + assert source.count("ctx.balance_consumer_is_due(") == 1 + assert '"%s", 3)' % route.token in source + assert "? (ctx.sum_component(" in source assert "ctx.record_scalar(" not in source assert route.token in source +def test_balance_due_contract_unions_consumers_and_ignores_static_false(t): + from pops.diagnostics import BalanceLedger + + P = t.Program("balance-due-contract") + U = typed_state(P, "blk") + route = BalanceLedger("mass").route_identity(U.block) + contract = _balance_due_contract( + route, + every(5, clock=P.clock), + when(False, clock=P.clock), + every(3, clock=P.clock), + ) + + assert contract.route(route.token).accepted_step_periods() == (3, 5) + false_only = _balance_due_contract(route, when(False, clock=P.clock)) + assert false_only.route(route.token).accepted_step_periods() == () + + +def test_record_balance_elides_native_collectives_without_a_consumer(t): + from pops.diagnostics import BalanceLedger + + P = t.Program("balance-without-consumer") + U = typed_state(P, "blk") + total = P.sum(U) + P.record_balance( + BalanceLedger("mass"), + storage_change=total, + outward_boundary_flux=total, + sources=total, + reflux=total, + projection=total, + ) + endpoint = typed_state(P, "blk", state_name="U").next + P.commit(endpoint, P.value("balance_next", U, at=endpoint.point)) + + source = emit_cpp_program(P) + + assert "ctx.balance_consumer_is_due(" not in source + assert "ctx.record_balance_term(" not in source + assert "(false) ? (ctx.sum_component(" in source + + +def test_record_balance_keeps_a_shared_reduction_unconditional(t): + from pops.diagnostics import BalanceLedger + + P = t.Program("balance-shared-reduction") + U = typed_state(P, "blk") + total = P.sum(U) + ledger = BalanceLedger("mass") + P.record_balance( + ledger, + storage_change=total, + outward_boundary_flux=total, + sources=total, + reflux=total, + projection=total, + ) + P.record_scalar("mass", total) + endpoint = typed_state(P, "blk", state_name="U").next + P.commit(endpoint, P.value("balance_next", U, at=endpoint.point)) + route = ledger.route_identity(U.block) + + source = emit_cpp_program( + P, + balance_due_contract=_balance_due_contract( + route, every(4, clock=P.clock) + ), + ) + reduction_line = next( + line for line in source.splitlines() if "ctx.sum_component(" in line + ) + + assert "? (ctx.sum_component(" not in reduction_line + assert source.count("ctx.record_balance_term(") == 5 + assert 'ctx.record_scalar("mass"' in source + + +def test_record_balance_physical_time_cadence_stays_conservatively_due(t): + from pops.diagnostics import BalanceLedger + + P = t.Program("balance-physical-cadence") + U = typed_state(P, "blk") + total = P.sum(U) + ledger = BalanceLedger("mass") + P.record_balance( + ledger, + storage_change=total, + outward_boundary_flux=total, + sources=total, + reflux=total, + projection=total, + ) + endpoint = typed_state(P, "blk", state_name="U").next + P.commit(endpoint, P.value("balance_next", U, at=endpoint.point)) + route = ledger.route_identity(U.block) + + source = emit_cpp_program( + P, + balance_due_contract=_balance_due_contract( + route, every_dt(0.1, clock=P.clock) + ), + ) + + assert source.count("ctx.balance_consumer_is_due(") == 1 + assert '"%s", 1)' % route.token in source + assert source.count("ctx.record_balance_term(") == 5 + + +def test_balance_consumer_without_a_program_producer_fails_before_codegen(t): + from pops.codegen.program_balance_due import validate_balance_due_contract + from pops.diagnostics import BalanceLedger + + P = t.Program("balance-missing-producer") + U = typed_state(P, "blk") + endpoint = typed_state(P, "blk", state_name="U").next + P.commit(endpoint, P.value("balance_next", U, at=endpoint.point)) + route = BalanceLedger("mass").route_identity(U.block) + contract = _balance_due_contract(route, every(2, clock=P.clock)) + + with pytest.raises( + ValueError, + match="Balance routes have no Program.record_balance producer", + ): + validate_balance_due_contract(P, contract) + + def test_record_balance_rejects_non_reduced_or_incomplete_evidence(t): from pops.diagnostics import BalanceLedger From b5a9ae749eddfc53ccfeecd3429a4031d06663b1 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 00:03:18 +0200 Subject: [PATCH 027/109] docs(output): document balance due fusion --- docs/design/exact-output-consumers.md | 29 +++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/docs/design/exact-output-consumers.md b/docs/design/exact-output-consumers.md index fc4304481..89046b252 100644 --- a/docs/design/exact-output-consumers.md +++ b/docs/design/exact-output-consumers.md @@ -460,14 +460,27 @@ The `pops.balance-term` namespace is reserved. Ordinary `Program.record_scalar(. the Python runtime diagnostic binding both reject it; generated `record_balance` code reaches a separate native sink that validates the route and canonical term before touching the mailbox. -`record_balance` is not currently gated by the matching Consumer cadence. Its five term-producing -Program reduction paths run whenever execution reaches the call, including every cadence/substep, -even if the `Balance` consumer is due only every N accepted steps. Use this explicit route with a -dense (every-invocation) balance cadence unless that collective cost is intentionally acceptable: -a sparse Consumer cadence does not save the upstream reductions. Scheduling only the five terminal -record nodes would not fix this, because their reduction inputs would still execute. A future -low-overhead sparse route therefore needs one typed due decision shared by the Program and -ConsumerGraph. +The resolved `ConsumerGraph` now compiles one immutable `BalanceDueContract` into the Program +artifact. For `every(n, clock=program.clock)`, the native Program queries the next outer accepted +macro-step before any balance reduction. Off-cadence sum/dot and scalar-arithmetic chains are +short-circuited, and the five terminal records are omitted; no Kokkos kernel, MPI collective or +Python callback is entered for that balance route. Multiple consumers of the same route are joined +by an OR of their exact accepted-step periods. `Always` and `when(True)` are period one, +`when(False)` contributes no occurrence, and a route with no consumer is compiled off. + +The compiler traces the complete reduction/scalar chain rather than scheduling only the terminal +records. If a value is also consumed by an ordinary Program diagnostic or another non-balance +operation, that shared producer remains unconditional so cadence fusion cannot change unrelated +semantics. A `Balance` consumer with no matching five-term `Program.record_balance` producer fails +before native code generation. Program stride/substeps use one attempt-local outer accepted-step +target, so every substep of one due public step sees the same decision and accumulates into the same +attempt mailbox. + +This first sparse cutover is exact only for accepted-step `every(n)` schedules. Physical-time +`every_dt`, `on_end`, and extension domains/triggers remain conservatively active for every Program +invocation; their consumer still publishes only when its own runtime schedule is due, but upstream +balance reductions are not yet skipped. This fallback can add work but cannot suppress required +evidence. This route is explicit evidence, not automatic numerical instrumentation: a Program that cannot produce its actual reflux or projection increment cannot declare `Balance`. In particular, the From 1e14b4bd9d17f0427bf8c6d7d53b9acbcc829b63 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 01:04:40 +0200 Subject: [PATCH 028/109] fix(ci): isolate the balance contract import leaf --- python/pops/_balance_contract.py | 91 ++++++++++++++++++++++ python/pops/codegen/program_balance_due.py | 2 +- python/pops/diagnostics/balance.py | 90 ++------------------- python/pops/time/_program/diagnostics.py | 2 +- 4 files changed, 99 insertions(+), 86 deletions(-) create mode 100644 python/pops/_balance_contract.py diff --git a/python/pops/_balance_contract.py b/python/pops/_balance_contract.py new file mode 100644 index 000000000..2fcf86671 --- /dev/null +++ b/python/pops/_balance_contract.py @@ -0,0 +1,91 @@ +"""Core typed identity shared by Program balance evidence and output consumers. + +This module deliberately lives outside :mod:`pops.diagnostics`: Program/codegen imports must not +make every PoPS test transitively depend on the public diagnostics package initializer. The public +``pops.diagnostics.BalanceLedger`` name is an alias of this exact class. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from pops.identity import Identity, make_identity + + +BALANCE_TERM_NAMES = ( + "storage_change", + "outward_boundary_flux", + "sources", + "reflux", + "projection", +) + + +def _canonical_name(value: Any, *, where: str) -> str: + if not isinstance(value, str) or not value or value.strip() != value: + raise TypeError("%s must be non-empty canonical text" % where) + return value + + +@dataclass(frozen=True, slots=True) +class BalanceLedger: + """Identity joining one Program-authored discrete balance to one consumer. + + The ledger does not contain values. :meth:`Program.record_balance` writes the five reduced + scalars into the current native step-attempt mailbox, while + :class:`pops.diagnostics.Balance` selects the same identity after that attempt has advanced + successfully. + """ + + name: str + identity: Identity = field(init=False) + __pops_ir_immutable__ = True + + def __post_init__(self) -> None: + name = _canonical_name(self.name, where="BalanceLedger.name") + object.__setattr__(self, "name", name) + object.__setattr__( + self, + "identity", + make_identity("balance-ledger", {"schema_version": 1, "name": name}), + ) + + def to_data(self) -> dict[str, Any]: + return { + "schema_version": 1, + "name": self.name, + "identity": self.identity.to_data(), + } + + def route_identity(self, block: Any) -> Identity: + from pops.problem.handles import BlockHandle + + if not isinstance(block, BlockHandle): + raise TypeError("balance ledger block must be a BlockHandle") + return make_identity( + "balance-ledger-route", + { + "schema_version": 1, + "ledger": self.identity.to_data(), + # Program records this route before Case resolution. Runtime block names are unique + # inside one Case/Program; the consumer separately carries the canonical block and + # state identity. + "runtime_block": block.local_id, + }, + ) + + +def balance_record_name(route: Any, term: Any) -> str: + """Return the reserved native Program diagnostic key for one exact term.""" + if ( + type(route) is not Identity + or route.domain != "balance-ledger-route" + or route.schema_version != 1 + ): + raise TypeError("balance route must be an exact balance-ledger-route Identity") + if term not in BALANCE_TERM_NAMES: + raise ValueError("unknown balance term %r" % (term,)) + return "pops.balance-term.v1:%s:%s" % (route.token, term) + + +__all__ = ["BALANCE_TERM_NAMES", "BalanceLedger", "balance_record_name"] diff --git a/python/pops/codegen/program_balance_due.py b/python/pops/codegen/program_balance_due.py index 0d4923f07..2459881bb 100644 --- a/python/pops/codegen/program_balance_due.py +++ b/python/pops/codegen/program_balance_due.py @@ -7,7 +7,7 @@ from types import MappingProxyType from typing import Any -from pops.diagnostics.balance import BALANCE_TERM_NAMES +from pops._balance_contract import BALANCE_TERM_NAMES from pops.identity import Identity from pops.output._balance_due_contract import BalanceDueContract from pops.time.values import ProgramValue diff --git a/python/pops/diagnostics/balance.py b/python/pops/diagnostics/balance.py index 5b9bfd173..b087a8145 100644 --- a/python/pops/diagnostics/balance.py +++ b/python/pops/diagnostics/balance.py @@ -1,87 +1,9 @@ -"""Typed identity shared by native Program balance evidence and output consumers.""" -from __future__ import annotations +"""Public balance diagnostic contract. -from dataclasses import dataclass, field -from typing import Any +The implementation lives in :mod:`pops._balance_contract` so native Program/codegen modules do not +depend on this package initializer. These aliases preserve the documented public import route. +""" -from pops.identity import Identity, make_identity +from pops._balance_contract import BALANCE_TERM_NAMES, BalanceLedger, balance_record_name - -BALANCE_TERM_NAMES = ( - "storage_change", - "outward_boundary_flux", - "sources", - "reflux", - "projection", -) - - -def _canonical_name(value: Any, *, where: str) -> str: - if not isinstance(value, str) or not value or value.strip() != value: - raise TypeError("%s must be non-empty canonical text" % where) - return value - - -@dataclass(frozen=True, slots=True) -class BalanceLedger: - """Identity joining one Program-authored discrete balance to one consumer. - - The ledger does not contain values. :meth:`Program.record_balance` writes the five - reduced scalars into the current native step-attempt mailbox, while - :class:`pops.diagnostics.Balance` selects the same identity after that attempt has - advanced successfully. - """ - - name: str - identity: Identity = field(init=False) - __pops_ir_immutable__ = True - - def __post_init__(self) -> None: - name = _canonical_name(self.name, where="BalanceLedger.name") - object.__setattr__(self, "name", name) - object.__setattr__( - self, - "identity", - make_identity("balance-ledger", {"schema_version": 1, "name": name}), - ) - - def to_data(self) -> dict[str, Any]: - return { - "schema_version": 1, - "name": self.name, - "identity": self.identity.to_data(), - } - - def route_identity(self, block: Any) -> Identity: - from pops.problem.handles import BlockHandle - - if not isinstance(block, BlockHandle): - raise TypeError("balance ledger block must be a BlockHandle") - return make_identity( - "balance-ledger-route", - { - "schema_version": 1, - "ledger": self.identity.to_data(), - # The Program records this route before Case resolution, whereas the - # consumer is resolved later. Runtime block names are unique inside one - # Case/Program, and the consumer quantity separately carries the complete - # canonical block/state identity. - "runtime_block": block.local_id, - }, - ) - - -def balance_record_name(route: Any, term: Any) -> str: - """Return the reserved native Program diagnostic key for one exact term.""" - if ( - type(route) is not Identity - or route.domain != "balance-ledger-route" - or route.schema_version != 1 - ): - raise TypeError("balance route must be an exact balance-ledger-route Identity") - if term not in BALANCE_TERM_NAMES: - raise ValueError("unknown balance term %r" % (term,)) - return "pops.balance-term.v1:%s:%s" % (route.token, term) - - -__all__ = ["BALANCE_TERM_NAMES", "BalanceLedger"] +__all__ = ["BALANCE_TERM_NAMES", "BalanceLedger", "balance_record_name"] diff --git a/python/pops/time/_program/diagnostics.py b/python/pops/time/_program/diagnostics.py index bc2158c3d..75d3ab00f 100644 --- a/python/pops/time/_program/diagnostics.py +++ b/python/pops/time/_program/diagnostics.py @@ -46,7 +46,7 @@ def record_balance( extrema/norm reductions, and rank-local runtime scalars are rejected. The five records are attempt-local: a rejected step or consumer rollback cannot leave evidence for a later sample. """ - from pops.diagnostics.balance import ( + from pops._balance_contract import ( BALANCE_TERM_NAMES, BalanceLedger, balance_record_name, From 5501456f13783df07c259f86497a25fdb29dfe15 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 01:14:11 +0200 Subject: [PATCH 029/109] fix(runtime): close sparse balance edge cases --- .../runtime/program/program_runtime_state.hpp | 41 +++++++++++++++++++ .../runtime/system/system_program_driver.hpp | 1 + src/runtime/amr/amr_system.cpp | 10 ++++- src/runtime/system/system_impl.hpp | 6 +++ src/runtime/system/system_io.cpp | 4 +- 5 files changed, 60 insertions(+), 2 deletions(-) diff --git a/include/pops/runtime/program/program_runtime_state.hpp b/include/pops/runtime/program/program_runtime_state.hpp index 815205379..e25835d8c 100644 --- a/include/pops/runtime/program/program_runtime_state.hpp +++ b/include/pops/runtime/program/program_runtime_state.hpp @@ -269,6 +269,15 @@ struct ProgramRuntimeState { /// balance code must not infer the public target from `macro_step()+1`. bool balance_due_window_active_ = false; int balance_due_target_step_ = 0; + /// Selective checkpoint reconstruction re-executes scientific Program code without accepting a + /// public step. Balance evidence is therefore compiled off for that replay: it must neither query + /// a nonexistent public-step due window nor populate the current accepted-attempt mailbox. + bool balance_replay_active_ = false; + /// A stride-held public step executes no Program work, so its exact discrete balance is the + /// additive identity for every route. These transient flags distinguish that valid zero from a + /// due Program that failed to publish all five terms; neither flag is checkpoint state. + bool balance_step_completed_ = false; + bool balance_program_was_due_ = false; /// Attempt-local identities of ProjectAndRecheck branches that actually executed. This report /// mailbox is cleared at attempt entry and consumed by the Python transaction coordinator before /// commit or rollback; it is deliberately not checkpoint or accepted scientific state. @@ -763,6 +772,13 @@ struct ProgramRuntimeState { step_balance_terms_.clear(); balance_due_window_active_ = false; balance_due_target_step_ = 0; + balance_step_completed_ = false; + balance_program_was_due_ = false; + } + + void complete_balance_step(bool program_was_due) noexcept { + balance_step_completed_ = true; + balance_program_was_due_ = program_was_due; } /// Return exactly the five native Program scalars recorded for one typed balance route during the @@ -774,6 +790,11 @@ struct ProgramRuntimeState { "sources", "reflux", "projection"}; require_balance_route(route, runtime + "::_accepted_balance_terms"); std::map result; + if (step_balance_terms_.empty() && balance_step_completed_ && !balance_program_was_due_) { + for (const char* term : kTerms) + result.emplace(term, Real(0)); + return result; + } for (const char* term : kTerms) { const std::string record = "pops.balance-term.v1:" + route + ":" + term; const auto found = step_balance_terms_.find(record); @@ -794,6 +815,8 @@ struct ProgramRuntimeState { void begin_balance_due_window(int accepted_macro_step, const std::string& runtime) { if (balance_due_window_active_) throw std::logic_error(runtime + " balance due window is already active"); + if (balance_replay_active_) + throw std::logic_error(runtime + " cannot enter a public-step window during balance replay"); if (accepted_macro_step < 0 || accepted_macro_step == std::numeric_limits::max()) throw std::overflow_error(runtime + " balance due target step is not representable"); balance_due_target_step_ = accepted_macro_step + 1; @@ -817,12 +840,30 @@ struct ProgramRuntimeState { end_balance_due_window(); } + template + void run_balance_replay(const std::string& runtime, Body&& body) { + if (balance_replay_active_) + throw std::logic_error(runtime + " balance replay is already active"); + if (balance_due_window_active_) + throw std::logic_error(runtime + " cannot enter balance replay inside a public-step window"); + balance_replay_active_ = true; + try { + std::forward(body)(); + } catch (...) { + balance_replay_active_ = false; + throw; + } + balance_replay_active_ = false; + } + bool balance_consumer_is_due(const std::string& contract, const std::string& route, int every_n, const std::string& runtime) const { require_balance_due_contract(contract, runtime + "::balance_consumer_is_due"); require_balance_route(route, runtime + "::balance_consumer_is_due"); if (every_n <= 0) throw std::invalid_argument(runtime + "::balance_consumer_is_due requires a positive period"); + if (balance_replay_active_) + return false; if (!balance_due_window_active_ || balance_due_target_step_ <= 0) throw std::logic_error(runtime + "::balance_consumer_is_due requires an active public-step window"); diff --git a/include/pops/runtime/system/system_program_driver.hpp b/include/pops/runtime/system/system_program_driver.hpp index ad9cdb7da..68afdfb6e 100644 --- a/include/pops/runtime/system/system_program_driver.hpp +++ b/include/pops/runtime/system/system_program_driver.hpp @@ -213,6 +213,7 @@ class SystemProgramDriver { // accepted_time + dt or window_start + effective_dt here would reintroduce a second authority. P->t = cadence.window_end; // clock ticks EVERY macro-step (held steps included), like native P->macro_step_++; + P->program_.complete_balance_step(cadence.due); } /// One macro-step of length @p dt through the installed whole-system Program. diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index 9eefff360..132f27592 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -442,6 +442,7 @@ struct AmrSystem::Impl { // One prepared endpoint owns facade, stages and serialized AMR accepted clocks. Do not recompute // it as either accepted_time + dt or window_start + effective_dt after Program execution. t = cadence.window_end; + program_.complete_balance_step(cadence.due); } struct AcceptedSnapshot { @@ -461,6 +462,8 @@ struct AmrSystem::Impl { int cadence_clock_restore_macro_step = 0; std::map program_diagnostics; std::map step_balance_terms; + bool balance_step_completed = false; + bool balance_program_was_due = false; pops::runtime::program::CacheManager cache; pops::runtime::program::HistoryManager history; pops::runtime::program::Profiler profiler; @@ -506,6 +509,8 @@ struct AmrSystem::Impl { cadence_clock_restore_macro_step = impl.program_.cadence_clock_restore_macro_step_; copy_value_map_into(program_diagnostics, impl.program_.diagnostics_); copy_value_map_into(step_balance_terms, impl.program_.step_balance_terms_); + balance_step_completed = impl.program_.balance_step_completed_; + balance_program_was_due = impl.program_.balance_program_was_due_; // AMR currently owns its native cache/history rings inside AmrRuntime. These two shared // ProgramRuntimeState containers are therefore empty on the AMR path, but retain their value // contract so a future target can populate them without weakening rollback semantics. @@ -537,6 +542,8 @@ struct AmrSystem::Impl { impl.program_.cadence_clock_restore_macro_step_ = cadence_clock_restore_macro_step; copy_value_map_into(impl.program_.diagnostics_, program_diagnostics); copy_value_map_into(impl.program_.step_balance_terms_, step_balance_terms); + impl.program_.balance_step_completed_ = balance_step_completed; + impl.program_.balance_program_was_due_ = balance_program_was_due; impl.program_.cache_ = cache; impl.program_.hist_ = history; impl.program_.profiler_ = profiler; @@ -4448,7 +4455,8 @@ int AmrSystem::rebuild_history_slots(const std::string& name, p_->program_.stride_, [imp](double dt, int cursor) { imp->macro_step_ = cursor; // ctx.macro_step() -> facade cursor -> regrid_if_due schedule imp->program_.last_dt_ = static_cast(dt); - imp->program_.step_(dt); + imp->program_.run_balance_replay("AmrSystem::rebuild_history_slots", + [&] { imp->program_.step_(dt); }); }); } catch (...) { p_->macro_step_ = m; diff --git a/src/runtime/system/system_impl.hpp b/src/runtime/system/system_impl.hpp index aea9dc2d0..13abe58bd 100644 --- a/src/runtime/system/system_impl.hpp +++ b/src/runtime/system/system_impl.hpp @@ -617,6 +617,8 @@ struct System::Impl { int cadence_clock_restore_macro_step; std::map program_diagnostics; std::map step_balance_terms; + bool balance_step_completed; + bool balance_program_was_due; pops::runtime::program::CacheManager cache; pops::runtime::program::HistoryManager history; pops::runtime::program::Profiler profiler; @@ -641,6 +643,8 @@ struct System::Impl { cadence_clock_restore_macro_step(impl.program_.cadence_clock_restore_macro_step_), program_diagnostics(impl.program_.diagnostics_), step_balance_terms(impl.program_.step_balance_terms_), + balance_step_completed(impl.program_.balance_step_completed_), + balance_program_was_due(impl.program_.balance_program_was_due_), cache(impl.program_.cache_), history(impl.program_.hist_), profiler(impl.program_.profiler_), @@ -673,6 +677,8 @@ struct System::Impl { impl.program_.cadence_clock_restore_macro_step_ = cadence_clock_restore_macro_step; impl.program_.diagnostics_ = program_diagnostics; impl.program_.step_balance_terms_ = step_balance_terms; + impl.program_.balance_step_completed_ = balance_step_completed; + impl.program_.balance_program_was_due_ = balance_program_was_due; impl.program_.cache_ = cache; impl.program_.hist_ = history; impl.program_.profiler_ = profiler; diff --git a/src/runtime/system/system_io.cpp b/src/runtime/system/system_io.cpp index 784d58471..ff2280e73 100644 --- a/src/runtime/system/system_io.cpp +++ b/src/runtime/system/system_io.cpp @@ -331,7 +331,9 @@ int System::rebuild_history_slots(const std::string& name, const std::vector newer; --j) { p_->program_.last_dt_ = dts[static_cast(j + 1)]; - p_->program_.step_(static_cast(dts[static_cast(j + 1)])); + p_->program_.run_balance_replay("System::rebuild_history_slots", [&] { + p_->program_.step_(static_cast(dts[static_cast(j + 1)])); + }); reconstructed[static_cast(j)] = p_->sp[owner].U; // deep copy the fresh owner state } From 105e3f55b72ab4698987dfcfa9dfd743e1d54d87 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 01:14:27 +0200 Subject: [PATCH 030/109] fix(runtime): keep zero-step output exact --- python/pops/runtime/_runtime_instance.py | 5 ++- .../runtime/test_runtime_instance_gate.py | 45 +++++++++++++++++++ 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/python/pops/runtime/_runtime_instance.py b/python/pops/runtime/_runtime_instance.py index 1da896c55..6a1d3a4b5 100644 --- a/python/pops/runtime/_runtime_instance.py +++ b/python/pops/runtime/_runtime_instance.py @@ -1403,8 +1403,9 @@ def _run(self, t_end: Any, *, max_steps: int = 1_000_000, "max_steps exhausted before t_end: " f"accepted {steps} step(s), reached t={native.time()!r}, " f"requested t_end={t_end!r}") - if steps == 0: - self._fire_consumers(at_end=True) + # A zero-step run has no accepted final occurrence. Its start consumers were already + # fired above; do not fabricate an AtEnd/Always/When/Every transaction at that same + # native state. close_live = getattr(self._publisher, "close_live_visualizations", None) if callable(close_live): close_live(manifest.run_identity) diff --git a/tests/python/unit/runtime/test_runtime_instance_gate.py b/tests/python/unit/runtime/test_runtime_instance_gate.py index f2649b8b5..7ab210aac 100644 --- a/tests/python/unit/runtime/test_runtime_instance_gate.py +++ b/tests/python/unit/runtime/test_runtime_instance_gate.py @@ -43,12 +43,15 @@ from pops.time import ( AcceptedStep, AdaptiveCFL, + Always, AtEnd, + AtStart, Clock, Every, ExternalTimeGrid, FixedDt, Schedule, + When, every_dt, ) from tests.python.support.native_execution_context import artifact_execution_context @@ -1124,6 +1127,48 @@ def test_run_fails_explicitly_when_max_steps_cannot_reach_t_end(tmp_path): assert tuple(tmp_path.glob("*.npz")) == () +@pytest.mark.parametrize( + "schedule", + ( + lambda clock: Schedule(Always(AcceptedStep(clock))), + lambda clock: Schedule(Every(AcceptedStep(clock), 1)), + lambda clock: Schedule(AtEnd(AcceptedStep(clock))), + lambda clock: Schedule(When(AcceptedStep(clock), True)), + ), +) +def test_zero_step_run_does_not_fabricate_an_accepted_consumer_occurrence( + tmp_path, schedule +): + plan, _, manifest = _with_graph(tmp_path, schedule=schedule) + runtime = RuntimeInstance(plan, executor=_Executor(plan)) + + report = runtime._run(t_end=0.0, max_steps=0) + + assert report.accepted_steps == 0 + assert ( + runtime.consumer_cursors.for_consumer(manifest.qualified_id).committed_samples + == 0 + ) + assert tuple(tmp_path.glob("*.npz")) == () + + +def test_zero_step_run_keeps_exactly_one_start_occurrence(tmp_path): + plan, _, manifest = _with_graph( + tmp_path, + schedule=lambda clock: Schedule(AtStart(AcceptedStep(clock))), + ) + runtime = RuntimeInstance(plan, executor=_Executor(plan)) + + report = runtime._run(t_end=0.0, max_steps=0) + + assert report.accepted_steps == 0 + assert ( + runtime.consumer_cursors.for_consumer(manifest.qualified_id).committed_samples + == 1 + ) + assert _published_times(tmp_path) == [0.0] + + def test_scientific_format_is_a_structural_provider_without_name_dispatch(tmp_path): plan, _, _ = _with_graph(tmp_path, output_format=_CustomNPZ) runtime = RuntimeInstance(plan, executor=_Executor(plan)) From b05e24f1a5ea59e7c86c71ab1da2bfedc4143f38 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 01:15:00 +0200 Subject: [PATCH 031/109] feat(time): author public program cadence --- CHANGELOG.md | 2 + python/pops/identity/semantic.py | 7 +- python/pops/runtime/_amr_system_program.py | 7 +- .../pops/runtime/_program_cadence_install.py | 39 +++++++ .../pops/runtime/_system_unified_install.py | 7 +- python/pops/time/_graph/program.py | 25 +++- python/pops/time/_program/api.py | 26 +++++ python/pops/time/_program/cadence.py | 59 ++++++++++ python/pops/time/_program/contract.py | 3 + python/pops/time/_program/graph_conversion.py | 1 + python/pops/time/_program/rebuild.py | 1 + python/pops/time/_program/serialization.py | 3 + .../python/unit/time/test_program_cadence.py | 110 ++++++++++++++++++ 13 files changed, 283 insertions(+), 7 deletions(-) create mode 100644 python/pops/runtime/_program_cadence_install.py create mode 100644 python/pops/time/_program/cadence.py create mode 100644 tests/python/unit/time/test_program_cadence.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 99374ebd9..f8ee5fb1b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning ### Changed +- `Program.cadence(substeps=..., stride=...)` now authors the native global cadence as immutable, + identity-bearing Program data and installs it before the Uniform or AMR runtime freezes. - AMR checkpoint capability reports now distinguish same-rank bit-identical replay from non-bit-identical rank-count rematerialization with Dense persisted histories, and state explicitly that `RegridOnRestart()` remains unsupported. The M3 gate now executes the persisted two-rank to diff --git a/python/pops/identity/semantic.py b/python/pops/identity/semantic.py index 0ee2d7893..5af8f8a57 100644 --- a/python/pops/identity/semantic.py +++ b/python/pops/identity/semantic.py @@ -121,6 +121,7 @@ def program_semantic_data(program: Any) -> dict[str, Any]: "history_persistence", "dt_bound", "step_transaction", + "cadence", } if not expected.issubset(serialized) or not set(serialized).issubset(expected | optional): raise TypeError("Program semantic projection received an unsupported IR schema") @@ -133,7 +134,11 @@ def program_semantic_data(program: Any) -> dict[str, Any]: "block_order": serialized["block_order"], } for key in ( - "histories", "history_contracts", "history_persistence", "step_transaction", + "histories", + "history_contracts", + "history_persistence", + "step_transaction", + "cadence", ): if key in serialized: result[key] = serialized[key] diff --git a/python/pops/runtime/_amr_system_program.py b/python/pops/runtime/_amr_system_program.py index 09d4b287d..442153a5e 100644 --- a/python/pops/runtime/_amr_system_program.py +++ b/python/pops/runtime/_amr_system_program.py @@ -41,6 +41,11 @@ def _finish_program_install(self, compiled: Any, so_path: Any, schema: Any, - (6) attach the exact typed StepTransactionPlan authored by the installed Program. """ if so_path is not None: + component = getattr(compiled, "program", None) + authored = getattr(component, "program", component) + from pops.runtime._program_cadence_install import install_program_cadence + + install_program_cadence(self, authored) self.install_program(so_path) # (5a) HISTORY-PERSISTENCE POLICIES (ADC-631, parity with the uniform step-5a): the compiled # Program records a per-ring persistence policy (Dense / Interval / Revolve) on @@ -54,8 +59,6 @@ def _finish_program_install(self, compiled: Any, so_path: Any, schema: Any, set_persistence( {name: policy for name, (_depth, policy) in persistence.items()}) self._install_program_params(compiled, schema, params) - component = getattr(compiled, "program", None) - authored = getattr(component, "program", component) self._step_strategy = getattr(authored, "_step_strategy", None) self._step_transaction_plan = ( authored.transaction_plan() if authored is not None else None) diff --git a/python/pops/runtime/_program_cadence_install.py b/python/pops/runtime/_program_cadence_install.py new file mode 100644 index 000000000..4c5b8511a --- /dev/null +++ b/python/pops/runtime/_program_cadence_install.py @@ -0,0 +1,39 @@ +"""Bind-time installation of the immutable cadence carried by a compiled Program.""" +from __future__ import annotations + +from typing import Any + + +def install_program_cadence(engine: Any, program: Any) -> None: + """Install one authenticated cadence before the native Program and runtime freeze.""" + from pops.time._program.cadence import ProgramCadence + from pops.time._program.contract import require_program + + require_program(program, exact=True, where="pops.bind Program cadence") + if getattr(program, "_compiled_detached", False) is not True \ + or getattr(program, "_frozen", False) is not True: + raise TypeError( + "pops.bind Program cadence requires the frozen compiled Program authority" + ) + contract = program.cadence_contract() + if type(contract) is not ProgramCadence: + raise TypeError("pops.bind Program cadence is not an exact ProgramCadence") + setter = getattr(engine, "set_program_cadence", None) + if not callable(setter): + raise RuntimeError("pops.bind runtime cannot install the authored Program cadence") + setter(contract.substeps, contract.stride) + + substeps = getattr(engine, "program_substeps", None) + stride = getattr(engine, "program_stride", None) + if not callable(substeps) or not callable(stride): + raise RuntimeError("pops.bind runtime cannot authenticate the installed Program cadence") + actual = (int(substeps()), int(stride())) + expected = (contract.substeps, contract.stride) + if actual != expected: + raise RuntimeError( + "pops.bind runtime Program cadence differs from the compiled contract: " + "expected=%r actual=%r" % (expected, actual) + ) + + +__all__ = ["install_program_cadence"] diff --git a/python/pops/runtime/_system_unified_install.py b/python/pops/runtime/_system_unified_install.py index eaba0848b..6f9f28d33 100644 --- a/python/pops/runtime/_system_unified_install.py +++ b/python/pops/runtime/_system_unified_install.py @@ -426,6 +426,11 @@ def _install_compiled(self, compiled=None, *, instances=None, params=None, aux=N # NATIVE mode (compiled=None) deliberately installs no temporal authority. The blocks are # inspectable spatial carriers, but step/advance fail closed until a Program is installed. if so_path is not None: + component = getattr(compiled, "program", None) + authored = getattr(component, "program", component) + from pops.runtime._program_cadence_install import install_program_cadence + + install_program_cadence(self, authored) self.install_program(so_path) # (5a) HISTORY-PERSISTENCE POLICIES (ADC-626): the compiled Program records a per-ring # persistence policy (Dense / Interval / Revolve) on program._history_persistence. Attach the @@ -441,8 +446,6 @@ def _install_compiled(self, compiled=None, *, instances=None, params=None, aux=N # (5b) Program carriers were emitted with neutral values. Always install the complete # BindSchema projection after loading, including declaration defaults. self._install_program_params(compiled, bind_schema, params) - component = getattr(compiled, "program", None) - authored = getattr(component, "program", component) self._step_strategy = getattr(authored, "_step_strategy", None) self._step_transaction_plan = ( authored.transaction_plan() if authored is not None else None) diff --git a/python/pops/time/_graph/program.py b/python/pops/time/_graph/program.py index 42e11d64b..c5253ba02 100644 --- a/python/pops/time/_graph/program.py +++ b/python/pops/time/_graph/program.py @@ -11,6 +11,7 @@ from pops.time._graph.nodes import NODE_TYPES from pops.time._graph.validation import validate_nodes from pops.time.points import Clock +from pops.time._program.cadence import ProgramCadence GRAPH_NODE_TYPES = (*NODE_TYPES, Branch, Loop) @@ -23,9 +24,17 @@ class ProgramGraph: name: str clocks: tuple[Clock, ...] nodes: tuple[Any, ...] + cadence: ProgramCadence graph_hash: str - def __init__(self, name: str, nodes: Any, *, clocks: Any = None) -> None: + def __init__( + self, + name: str, + nodes: Any, + *, + clocks: Any = None, + cadence: Any = None, + ) -> None: object.__setattr__(self, "name", nonempty(name, where="ProgramGraph name")) frozen_nodes = tuple(nodes) if any(type(node) not in GRAPH_NODE_TYPES for node in frozen_nodes): @@ -38,6 +47,15 @@ def __init__(self, name: str, nodes: Any, *, clocks: Any = None) -> None: raise ValueError("ProgramGraph clocks must be unique") object.__setattr__(self, "clocks", declared) object.__setattr__(self, "nodes", frozen_nodes) + if cadence is None: + cadence = ProgramCadence() + elif isinstance(cadence, dict): + cadence = ProgramCadence.from_data(cadence) + if type(cadence) is not ProgramCadence: + raise TypeError( + "ProgramGraph cadence must be exact ProgramCadence data" + ) + object.__setattr__(self, "cadence", cadence) available: dict[int, Any] = {} validate_nodes(self.nodes, self.clocks, available, where="ProgramGraph") payload = json.dumps(self.to_data(), sort_keys=True, separators=(",", ":")) @@ -52,13 +70,16 @@ def ref(self, node: Any) -> ValueRef: return ValueRef(node.node_id) def to_data(self) -> dict[str, Any]: - return { + result = { "schema_version": 1, "kind": "pops.program-graph", "name": self.name, "clocks": [clock.to_data() for clock in self.clocks], "nodes": [node.to_data() for node in self.nodes], } + if not self.cadence.is_default: + result["cadence"] = self.cadence.to_data() + return result __all__ = ["ProgramGraph"] diff --git a/python/pops/time/_program/api.py b/python/pops/time/_program/api.py index 916f8df2b..72b107213 100644 --- a/python/pops/time/_program/api.py +++ b/python/pops/time/_program/api.py @@ -13,6 +13,7 @@ from pops.model.ownership import OwnerKind, OwnerPath from pops.time._program.contract import register_program_type +from pops.time._program.cadence import ProgramCadence from pops.time._program.authoring import _ProgramAuthoring from pops.time._program.condensed import _ProgramCondensed from pops.time._program.operations import _ProgramCore @@ -118,6 +119,9 @@ def __init__(self, name: Any) -> None: # ADC-666: explicit attempt controller. Runtime kwargs are validated against this descriptor; # a run-time CFL/dt/error-control option never silently selects a strategy. self._step_strategy = None + # The default executes once per accepted macro-step. A non-default cadence is an authored, + # immutable part of the Program identity and is installed before the runtime freezes. + self._cadence = None self._transaction_stores = ALL_PROVISIONAL_STORES self._acceptance_guards = () # ADC-563 freeze: a Program is MUTABLE while authored and FROZEN by pops.compile. After @@ -218,6 +222,28 @@ def step_strategy( self._transaction_stores = stores return self + def cadence(self, *, substeps: Any = 1, stride: Any = 1) -> Any: + """Declare the global Program cadence once, before compile. + + ``stride`` accumulates accepted macro-step intervals and executes the Program when the + window closes. ``substeps`` divides that complete window into exact Program executions. + Off-cadence accepted steps sample-and-hold the last Program state. + """ + self._guard_mutable("set Program cadence") + if self._cadence is not None: + raise ValueError("Program.cadence may be declared only once") + self._cadence = ProgramCadence(substeps=substeps, stride=stride) + return self + + def cadence_contract(self) -> ProgramCadence: + """Return the immutable authored cadence, defaulting to one execution per macro-step.""" + cadence = self._cadence + if cadence is None: + return ProgramCadence() + if type(cadence) is not ProgramCadence: + raise TypeError("Program carries an invalid cadence contract") + return cadence + def _register_acceptance_guard(self, guard: AcceptanceGuard) -> None: self._guard_mutable("register acceptance guard %r" % guard.name) if any(existing.name == guard.name for existing in self._acceptance_guards): diff --git a/python/pops/time/_program/cadence.py b/python/pops/time/_program/cadence.py new file mode 100644 index 000000000..ebea4a646 --- /dev/null +++ b/python/pops/time/_program/cadence.py @@ -0,0 +1,59 @@ +"""Immutable macro-step cadence authored by :class:`pops.time.Program`.""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +def _positive_int(value: Any, *, where: str) -> int: + if isinstance(value, bool) or type(value) is not int: + raise TypeError("%s must be an exact int" % where) + if value < 1: + raise ValueError("%s must be >= 1" % where) + return value + + +@dataclass(frozen=True, slots=True) +class ProgramCadence: + """Exact global Program executions within an accepted macro-step window.""" + + substeps: int = 1 + stride: int = 1 + + def __post_init__(self) -> None: + object.__setattr__( + self, + "substeps", + _positive_int(self.substeps, where="Program cadence substeps"), + ) + object.__setattr__( + self, + "stride", + _positive_int(self.stride, where="Program cadence stride"), + ) + + @property + def is_default(self) -> bool: + return self.substeps == 1 and self.stride == 1 + + def to_data(self) -> dict[str, int]: + return { + "schema_version": 1, + "substeps": self.substeps, + "stride": self.stride, + } + + @classmethod + def from_data(cls, data: Any) -> ProgramCadence: + if type(data) is not dict or set(data) != { + "schema_version", + "substeps", + "stride", + }: + raise TypeError("Program cadence data must contain the exact v1 schema") + if type(data["schema_version"]) is not int or data["schema_version"] != 1: + raise ValueError("Program cadence schema_version must be 1") + return cls(data["substeps"], data["stride"]) + + +__all__ = ["ProgramCadence"] diff --git a/python/pops/time/_program/contract.py b/python/pops/time/_program/contract.py index ff3a528a6..4e39deeaf 100644 --- a/python/pops/time/_program/contract.py +++ b/python/pops/time/_program/contract.py @@ -90,6 +90,7 @@ class _ProgramBase: _capture_source: bool _provenance_context: Any _step_strategy: Any + _cadence: Any _transaction_stores: Any _acceptance_guards: tuple _frozen: bool @@ -144,6 +145,8 @@ def _region_for_block(self, block: Any) -> int: ... def _allow_region_capture(self, source: int, destination: int) -> None: ... def _register_acceptance_guard(self, guard: Any) -> None: ... def transaction_plan(self) -> Any: ... + def cadence(self, *, substeps: Any = 1, stride: Any = 1) -> Any: ... + def cadence_contract(self) -> Any: ... def state(self, state: Any, *, clock: Any = None) -> Any: ... def synchronize( self, value: Any, *, at: Any, relation: Any, name: Any = None diff --git a/python/pops/time/_program/graph_conversion.py b/python/pops/time/_program/graph_conversion.py index e599da864..3b14141b9 100644 --- a/python/pops/time/_program/graph_conversion.py +++ b/python/pops/time/_program/graph_conversion.py @@ -347,6 +347,7 @@ def convert_values(values: Any) -> list[Any]: detached.name, nodes, clocks=_declared_clocks(nodes, detached.clock), + cadence=detached.cadence_contract(), ) # Detachment and graph conversion are read-only; authoring identity remains stable. if detached._ir_hash() != program._ir_hash(): diff --git a/python/pops/time/_program/rebuild.py b/python/pops/time/_program/rebuild.py index 0a9fccc9d..d21058438 100644 --- a/python/pops/time/_program/rebuild.py +++ b/python/pops/time/_program/rebuild.py @@ -84,6 +84,7 @@ def _keep_registry(_owner: Any) -> bool: object.__setattr__(out, "clock", Clock("macro", owner=out.owner_path)) out.dt = self.dt out._step_strategy = getattr(self, "_step_strategy", None) + out._cadence = getattr(self, "_cadence", None) out._transaction_stores = tuple(getattr(self, "_transaction_stores", ())) out._acceptance_guards = tuple(getattr(self, "_acceptance_guards", ())) if project_states and (self._dt_bound is not None or out._acceptance_guards): diff --git a/python/pops/time/_program/serialization.py b/python/pops/time/_program/serialization.py index e74f4a736..095c7c7b6 100644 --- a/python/pops/time/_program/serialization.py +++ b/python/pops/time/_program/serialization.py @@ -213,6 +213,9 @@ def _serialize(self, *, include_provenance: bool = True) -> dict[str, Any]: transaction = self.transaction_plan() if transaction is not None: result["step_transaction"] = transaction.to_data() + cadence = self.cadence_contract() + if not cadence.is_default: + result["cadence"] = cadence.to_data() if self._histories: result["histories"] = [ { diff --git a/tests/python/unit/time/test_program_cadence.py b/tests/python/unit/time/test_program_cadence.py new file mode 100644 index 000000000..726df656f --- /dev/null +++ b/tests/python/unit/time/test_program_cadence.py @@ -0,0 +1,110 @@ +"""Public immutable Program cadence and its bind-time transport.""" +from __future__ import annotations + +import pytest + +from pops.identity.semantic import semantic_identity_of +from pops.runtime._program_cadence_install import install_program_cadence +from pops.time import Program +from pops.time._program.cadence import ProgramCadence +from pops.time._program.detach import detach_compiled_program + + +def test_program_cadence_is_single_declaration_exact_positive_and_identity_bearing(): + baseline = Program("baseline") + configured = Program("configured") + + assert "cadence" not in baseline._serialize(include_provenance=False) + assert "cadence" not in baseline.to_graph().to_data() + assert configured.cadence(substeps=2, stride=3) is configured + assert configured.cadence_contract().to_data() == { + "schema_version": 1, + "substeps": 2, + "stride": 3, + } + assert configured._serialize(include_provenance=False)["cadence"] == { + "schema_version": 1, + "substeps": 2, + "stride": 3, + } + assert configured.to_graph().to_data()["cadence"] == { + "schema_version": 1, + "substeps": 2, + "stride": 3, + } + assert configured._ir_hash() != baseline._ir_hash() + assert configured.to_graph().graph_hash != baseline.to_graph().graph_hash + assert semantic_identity_of(program=configured) != semantic_identity_of(program=baseline) + + with pytest.raises(ValueError, match="only once"): + configured.cadence(stride=4) + + for name, kwargs in ( + ("bool substeps", {"substeps": True, "stride": 1}), + ("bool stride", {"substeps": 1, "stride": False}), + ("float stride", {"substeps": 1, "stride": 2.0}), + ): + candidate = Program(name) + with pytest.raises(TypeError, match="exact int"): + candidate.cadence(**kwargs) + for name, kwargs in ( + ("zero substeps", {"substeps": 0, "stride": 1}), + ("zero stride", {"substeps": 1, "stride": 0}), + ): + candidate = Program(name) + with pytest.raises(ValueError, match=">= 1"): + candidate.cadence(**kwargs) + with pytest.raises(TypeError, match="exact v1 schema"): + ProgramCadence.from_data( + type("CadenceDict", (dict,), {})( + schema_version=1, substeps=1, stride=2 + ) + ) + with pytest.raises(ValueError, match="schema_version"): + ProgramCadence.from_data( + {"schema_version": True, "substeps": 1, "stride": 2} + ) + + +def test_compiled_detachment_preserves_cadence_and_freeze_refuses_mutation(): + authored = Program("detached-cadence").cadence(stride=3) + detached = detach_compiled_program(authored) + + assert detached is not authored + assert detached.cadence_contract() == authored.cadence_contract() + assert detached._ir_hash() == authored._ir_hash() + with pytest.raises(RuntimeError, match="frozen"): + detached.cadence(stride=4) + + +class _CadenceEngine: + def __init__(self, *, lie: bool = False) -> None: + self.calls = [] + self.substeps = 1 + self.stride = 1 + self.lie = lie + + def set_program_cadence(self, substeps, stride): + self.calls.append((substeps, stride)) + self.substeps = substeps + self.stride = stride + + def program_substeps(self): + return self.substeps + + def program_stride(self): + return self.stride + int(self.lie) + + +def test_bind_installs_and_authenticates_only_the_frozen_compiled_cadence(): + authored = Program("install-cadence").cadence(substeps=2, stride=3) + detached = detach_compiled_program(authored) + engine = _CadenceEngine() + + install_program_cadence(engine, detached) + + assert engine.calls == [(2, 3)] + with pytest.raises(TypeError, match="frozen compiled Program"): + install_program_cadence(_CadenceEngine(), authored) + with pytest.raises(RuntimeError, match="differs from the compiled contract"): + install_program_cadence(_CadenceEngine(lie=True), detached) From 5a730536152248ec0d79a32677c4024649e42abd Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 01:15:48 +0200 Subject: [PATCH 032/109] test(runtime): prove sparse balance edge cases --- .../amr/test_amr_system_contract.cpp | 40 +++++++++ .../runtime/test_program_runtime.cpp | 84 +++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/tests/cpp/integration/amr/test_amr_system_contract.cpp b/tests/cpp/integration/amr/test_amr_system_contract.cpp index a90f6eda1..896db5cac 100644 --- a/tests/cpp/integration/amr/test_amr_system_contract.cpp +++ b/tests/cpp/integration/amr/test_amr_system_contract.cpp @@ -436,6 +436,46 @@ TEST(test_amr_system_contract, VariableDtStrideUsesOneExactPublicWindow) { EXPECT_DOUBLE_EQ(system.program_cadence_window_start_time(), 0.0); } +TEST(test_amr_system_contract, StrideHeldStepPublishesTheExactZeroBalance) { +#if defined(POPS_HAS_KOKKOS) + Kokkos::ScopeGuard guard; +#endif + AmrSystemConfig cfg; + cfg.n = 4; + cfg.L = 1.0; + cfg.regrid_every = 0; + cfg.periodicity = {true, true}; + + AmrSystem system(cfg); + system.add_block("tracer", exb_spec(), "none", "rusanov", "conservative", "explicit", 1); + system.install_program_step([](double) {}); + system.set_program_cadence(/*substeps=*/1, /*stride=*/2); + system.begin_step_transaction(); + system.step(0.1); + + const std::string route = "pops.balance-ledger-route.v1:sha256:" + std::string(64, '8'); + const auto balance = system.accepted_balance_terms(route); + EXPECT_EQ(balance.size(), 5u); + for (const auto& [name, value] : balance) { + EXPECT_FALSE(name.empty()); + EXPECT_DOUBLE_EQ(value, 0.0); + } + system.commit_step_transaction(); + system.finalize_step_transaction(); + + system.begin_step_transaction(); + system.step(0.1); + system.rollback_step_transaction(); + system.begin_step_transaction(); + const auto restored = system.accepted_balance_terms(route); + EXPECT_EQ(restored.size(), 5u); + for (const auto& [name, value] : restored) { + EXPECT_FALSE(name.empty()); + EXPECT_DOUBLE_EQ(value, 0.0); + } + system.rollback_step_transaction(); +} + TEST(test_amr_system_contract, CadenceRestoreRejectsClockDriftWithoutMutatingAcceptedState) { #if defined(POPS_HAS_KOKKOS) Kokkos::ScopeGuard guard; diff --git a/tests/cpp/integration/runtime/test_program_runtime.cpp b/tests/cpp/integration/runtime/test_program_runtime.cpp index 122aa7940..cb9c67fbd 100644 --- a/tests/cpp/integration/runtime/test_program_runtime.cpp +++ b/tests/cpp/integration/runtime/test_program_runtime.cpp @@ -139,6 +139,31 @@ TEST(ProgramRuntime, BalanceDueWindowUsesTheOuterAcceptedStepAndCleansUpOnFailur EXPECT_THROW((void)state.balance_consumer_is_due(contract, route, 4, "test"), std::logic_error); } +TEST(ProgramRuntime, SelectiveReplayCompilesBalanceOffAndRestoresTheGuard) { + runtime::program::ProgramRuntimeState state; + const std::string contract = "pops.balance-due-contract.v1:sha256:" + std::string(64, '3'); + const std::string route = "pops.balance-ledger-route.v1:sha256:" + std::string(64, '4'); + + EXPECT_THROW((void)state.balance_consumer_is_due(contract, route, 2, "test"), std::logic_error); + state.run_balance_replay("test", [&] { + EXPECT_FALSE(state.balance_consumer_is_due(contract, route, 2, "test")); + EXPECT_THROW((void)state.balance_consumer_is_due("forged", route, 2, "test"), + std::invalid_argument); + EXPECT_THROW((void)state.balance_consumer_is_due(contract, route, 0, "test"), + std::invalid_argument); + EXPECT_THROW(state.run_balance_replay("nested", [] {}), std::logic_error); + EXPECT_THROW(state.run_balance_due_window(1, "nested", [] {}), std::logic_error); + }); + EXPECT_THROW((void)state.balance_consumer_is_due(contract, route, 2, "test"), std::logic_error); + state.run_balance_due_window(1, "test", [&] { + EXPECT_THROW(state.run_balance_replay("window", [] {}), std::logic_error); + }); + + EXPECT_THROW(state.run_balance_replay("test", [] { throw std::runtime_error("replay failed"); }), + std::runtime_error); + EXPECT_THROW((void)state.balance_consumer_is_due(contract, route, 2, "test"), std::logic_error); +} + TEST(ProgramRuntime, ReplayAuthorityRequiresAnArtifactAndAnExactRingDepthPair) { runtime::program::ProgramRuntimeState state; state.history_replay_authorities_ = {{"gas.previous", 3}}; @@ -320,6 +345,65 @@ TEST(ProgramRuntime, GlobalCadencePublishesExactSubstepAndStrideWindowTimes) { EXPECT_DOUBLE_EQ(catchup.program_cadence_window_start_time(), 0.0); } +TEST(ProgramRuntime, StrideHeldStepsPublishTheExactZeroBalance) { +#if defined(POPS_HAS_KOKKOS) + ensure_kokkos(); +#endif + SystemConfig config; + config.n = 4; + config.L = 1.0; + config.periodicity = {true, true}; + + System system(config); + runtime::program::ProgramContext context(&system); + const std::string route = "pops.balance-ledger-route.v1:sha256:" + std::string(64, '7'); + const std::array, 5> records{{ + {"storage_change", 1.0}, + {"outward_boundary_flux", 2.0}, + {"sources", 3.0}, + {"reflux", 4.0}, + {"projection", 5.0}, + }}; + context.install([&](double) { + for (const auto& [name, value] : records) + context.record_balance_term(route, name, value); + }); + system.set_program_cadence(/*substeps=*/1, /*stride=*/3); + + const auto step_and_read = [&]() { + system.begin_step_transaction(); + system.step(0.1); + const auto balance = system.accepted_balance_terms(route); + system.commit_step_transaction(); + system.finalize_step_transaction(); + return balance; + }; + + for (int held = 0; held < 2; ++held) { + const auto balance = step_and_read(); + ASSERT_EQ(balance.size(), records.size()); + for (const auto& [name, _value] : records) + EXPECT_DOUBLE_EQ(balance.at(name), 0.0); + } + + system.begin_step_transaction(); + system.step(0.1); + const auto rejected_due = system.accepted_balance_terms(route); + for (const auto& [name, value] : records) + EXPECT_DOUBLE_EQ(rejected_due.at(name), value); + system.rollback_step_transaction(); + system.begin_step_transaction(); + const auto restored_held = system.accepted_balance_terms(route); + for (const auto& [name, _value] : records) + EXPECT_DOUBLE_EQ(restored_held.at(name), 0.0); + system.rollback_step_transaction(); + + const auto due = step_and_read(); + ASSERT_EQ(due.size(), records.size()); + for (const auto& [name, value] : records) + EXPECT_DOUBLE_EQ(due.at(name), value); +} + TEST(ProgramRuntime, CadenceUsesThePreparedFacadeEndpointWhenFloatingPointAdditionIsNonAssociative) { #if defined(POPS_HAS_KOKKOS) From 2f92824446028aac5f051416d7fcd59391f7f0c0 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 01:15:55 +0200 Subject: [PATCH 033/109] fix(output): bound native balance cadence periods --- python/pops/output/_balance_due_contract.py | 10 +++++- .../python/unit/time/test_time_ops_polish.py | 34 +++++++++++++------ 2 files changed, 33 insertions(+), 11 deletions(-) diff --git a/python/pops/output/_balance_due_contract.py b/python/pops/output/_balance_due_contract.py index b19711fc3..25919fb86 100644 --- a/python/pops/output/_balance_due_contract.py +++ b/python/pops/output/_balance_due_contract.py @@ -9,6 +9,9 @@ from pops.time._schedule.domains import AcceptedStep +_MAX_NATIVE_ACCEPTED_STEP = (1 << 31) - 1 + + def _identity(value: Any, domain: str, *, where: str) -> Identity: if type(value) is not Identity or value.domain != domain or value.schema_version != 1: raise TypeError("%s must be an exact version-1 %s Identity" % (where, domain)) @@ -94,7 +97,12 @@ def accepted_step_periods(self) -> tuple[int, ...]: return (1,) trigger = schedule.trigger if type(trigger) is Every: - periods.append(trigger.n) + # The native facade's public macro-step is a signed 32-bit ``int`` and rejects + # overflow before increment. A larger positive period can therefore never fire in + # any representable run; omit it instead of emitting an implementation-defined C++ + # narrowing conversion. + if trigger.n <= _MAX_NATIVE_ACCEPTED_STEP: + periods.append(trigger.n) elif type(trigger) is Always: periods.append(1) elif type(trigger) is When and type(trigger.condition) is bool: diff --git a/tests/python/unit/time/test_time_ops_polish.py b/tests/python/unit/time/test_time_ops_polish.py index 089ee956c..d3f8a996c 100644 --- a/tests/python/unit/time/test_time_ops_polish.py +++ b/tests/python/unit/time/test_time_ops_polish.py @@ -332,9 +332,12 @@ def test_record_scalar_rejects_non_scalar_and_bad_name(t): def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): + from pops._balance_contract import BalanceLedger as CoreBalanceLedger from pops.diagnostics import BalanceLedger from pops.diagnostics.balance import BALANCE_TERM_NAMES, balance_record_name + assert BalanceLedger is CoreBalanceLedger + P = t.Program("p") U = typed_state(P, "blk") total = P.sum(U) @@ -349,7 +352,8 @@ def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): ) route = ledger.route_identity(U.block) assert tuple(record.attrs["diagnostic"] for record in records) == tuple( - balance_record_name(route, term) for term in BALANCE_TERM_NAMES) + balance_record_name(route, term) for term in BALANCE_TERM_NAMES + ) endpoint = typed_state(P, "blk", state_name="U").next P.commit(endpoint, P.value("balance_next", U, at=endpoint.point)) contract = _balance_due_contract(route, every(3, clock=P.clock)) @@ -361,6 +365,15 @@ def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): assert "ctx.record_scalar(" not in source assert route.token in source + unreachable_source = emit_cpp_program( + P, + balance_due_contract=_balance_due_contract( + route, every(1 << 31, clock=P.clock) + ), + ) + assert "2147483648" not in unreachable_source + assert "ctx.balance_consumer_is_due(" not in unreachable_source + def test_balance_due_contract_unions_consumers_and_ignores_static_false(t): from pops.diagnostics import BalanceLedger @@ -379,6 +392,13 @@ def test_balance_due_contract_unions_consumers_and_ignores_static_false(t): false_only = _balance_due_contract(route, when(False, clock=P.clock)) assert false_only.route(route.token).accepted_step_periods() == () + native_boundary = _balance_due_contract( + route, + every((1 << 31) - 1, clock=P.clock), + every(1 << 31, clock=P.clock), + ) + assert native_boundary.route(route.token).accepted_step_periods() == ((1 << 31) - 1,) + def test_record_balance_elides_native_collectives_without_a_consumer(t): from pops.diagnostics import BalanceLedger @@ -426,13 +446,9 @@ def test_record_balance_keeps_a_shared_reduction_unconditional(t): source = emit_cpp_program( P, - balance_due_contract=_balance_due_contract( - route, every(4, clock=P.clock) - ), - ) - reduction_line = next( - line for line in source.splitlines() if "ctx.sum_component(" in line + balance_due_contract=_balance_due_contract(route, every(4, clock=P.clock)), ) + reduction_line = next(line for line in source.splitlines() if "ctx.sum_component(" in line) assert "? (ctx.sum_component(" not in reduction_line assert source.count("ctx.record_balance_term(") == 5 @@ -460,9 +476,7 @@ def test_record_balance_physical_time_cadence_stays_conservatively_due(t): source = emit_cpp_program( P, - balance_due_contract=_balance_due_contract( - route, every_dt(0.1, clock=P.clock) - ), + balance_due_contract=_balance_due_contract(route, every_dt(0.1, clock=P.clock)), ) assert source.count("ctx.balance_consumer_is_due(") == 1 From 57b36483d7d89525610b5d8d3bb934d824fc9622 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 01:16:02 +0200 Subject: [PATCH 034/109] test(restart): replay balance programs selectively --- .../io/test_amr_history_checkpoint.py | 99 ++++++++++++++++++- ...st_uniform_selective_history_checkpoint.py | 48 ++++++++- 2 files changed, 140 insertions(+), 7 deletions(-) diff --git a/tests/python/integration/io/test_amr_history_checkpoint.py b/tests/python/integration/io/test_amr_history_checkpoint.py index e85ef4339..a8fde2d81 100644 --- a/tests/python/integration/io/test_amr_history_checkpoint.py +++ b/tests/python/integration/io/test_amr_history_checkpoint.py @@ -126,7 +126,13 @@ def _ab2_program(model, name="adc631_ckpt_ab2"): return P -def _state3_program(model, name="adc631_ckpt_state3", *, step_strategy=None): +def _state3_program( + model, + name="adc631_ckpt_state3", + *, + step_strategy=None, + balance_replay_proof=False, +): """A 3-slot STATE ring (max lag 2, Interval(2) -> stores slots {0,2}, replays slot 1). The commit is the strictly affine recurrence U^{n+1} = U^n + dt*_C*U^n -- it depends only on U^n, @@ -143,11 +149,59 @@ def _state3_program(model, name="adc631_ckpt_state3", *, step_strategy=None): # Strictly affine growth (reads U.n only), + a zero-weight prev(2) read that declares the 3-slot # ring without breaking the single-step reconstructability of the replay. nxt = P.value("Un", U.n + P.dt * _C * U.n + 0.0 * U.prev(2), at=U.next.point) + balance_due_contract = None + if balance_replay_proof: + from pops.diagnostics import BalanceLedger + from pops.identity import make_identity + from pops.output._balance_due_contract import ( + BalanceDueConsumer, + BalanceDueContract, + BalanceDueRoute, + ) + + total = P.sum(U) + ledger = BalanceLedger("amr-selective-replay") + P.record_balance( + ledger, + storage_change=total, + outward_boundary_flux=0.0 * total, + sources=0.0 * total, + reflux=0.0 * total, + projection=0.0 * total, + ) + route = ledger.route_identity(U.block) + balance_due_contract = BalanceDueContract( + make_identity("consumer-graph", {"test": "amr-selective-replay"}), + ( + BalanceDueRoute( + route, + ( + BalanceDueConsumer( + make_identity( + "consumer-manifest", + {"test": "amr-selective-replay"}, + ), + pops.time.every(2, clock=P.clock), + ), + ), + ), + ), + ) P.commit(U.next, nxt) P.step_strategy(pops.time.FixedDt(DT) if step_strategy is None else step_strategy) + if balance_due_contract is not None: + return P, balance_due_contract return P +def _state3_balance_program(model): + return _state3_program( + model, + name="adc686_ckpt_state3_balance", + balance_replay_proof=True, + ) + + def _state5_program(model, name="adc631_ckpt_state5"): """A 5-slot strictly affine ring with two independently replayed Interval(2) gaps.""" P = pops.Program(name) @@ -221,8 +275,17 @@ def _build(program_factory, regrid_every=2, program_cadence=None): "test_amr_history_checkpoint requires install_program/history_names bindings" ) model = _passive_source_model("%s_model" % program_factory.__name__.lstrip("_")) - program = program_factory(model) - compiled = compile_problem(model=model, time=program, target="amr_system") + authored = program_factory(model) + if isinstance(authored, tuple): + program, balance_due_contract = authored + else: + program, balance_due_contract = authored, None + compiled = compile_problem( + model=model, + time=program, + target="amr_system", + balance_due_contract=balance_due_contract, + ) block_cm = compile_block_model(model, target="amr_system") amr.add_equation( "blk", @@ -377,6 +440,35 @@ def test_state3_interval_replay_bit_identical(): ) +def test_state3_selective_replay_compiles_balance_off(): + print("== (2b) selective replay re-steps a Balance Program outside a public-step window ==") + out, err = _run_case( + _state3_balance_program, + nsteps=6, + half=3, + label="state3-balance", + regrid_every=0, + ) + assert out is not None, err + ref, got, cont_rings, rest_rings, stored_info, report = out + chk( + bool(stored_info) + and all( + requested == stored and len(stored) < depth and mode == "policy" and fp == [] + for depth, requested, stored, mode, fp in stored_info.values() + ), + "the Balance Program retains selective storage and therefore exercises replay", + ) + chk( + report is not None and any(h["recomputed_slots"] >= 1 for h in report.histories), + "restart re-executed the compiled Balance Program for an omitted slot", + ) + chk( + _rings_equal(cont_rings, rest_rings) and np.array_equal(ref, got), + "Balance is compiled off only during replay; restart and continuation remain bit-identical", + ) + + def test_state3_replay_window_straddling_regrid_bit_identical(): print("== (3) ckpt at m=6 straddles regrid step 4 -> explicit dense safety storage ==") out, err = _run_case(_state3_program, nsteps=10, half=6, label="straddle", regrid_every=4) @@ -522,6 +614,7 @@ def accepted_levels(system): def main(): test_ab2_dense_checkpoint_bit_identical() test_state3_interval_replay_bit_identical() + test_state3_selective_replay_compiles_balance_off() test_state3_replay_window_straddling_regrid_bit_identical() test_state5_multiple_anchor_gaps_replay_by_index_bit_identical() test_amr_variable_dt_stride_checkpoint_closes_like_continuous_run() diff --git a/tests/python/integration/io/test_uniform_selective_history_checkpoint.py b/tests/python/integration/io/test_uniform_selective_history_checkpoint.py index cd2a46edd..98b3e7a36 100644 --- a/tests/python/integration/io/test_uniform_selective_history_checkpoint.py +++ b/tests/python/integration/io/test_uniform_selective_history_checkpoint.py @@ -79,11 +79,46 @@ def _program(model): - """Five-slot affine state history with two independently replayable gaps.""" + """Five-slot affine history plus a sparse Balance producer guarded during replay.""" + from pops.diagnostics import BalanceLedger + from pops.output._balance_due_contract import ( + BalanceDueConsumer, + BalanceDueContract, + BalanceDueRoute, + ) + program = pops.Program("uniform_selective_state5") _case, states = program_states(program, model, ("blk",)) state = states["blk"] program.keep_history(state, depth=4, checkpoint_policy=Interval(2)) + total = program.sum(state) + ledger = BalanceLedger("uniform-selective-replay") + program.record_balance( + ledger, + storage_change=total, + outward_boundary_flux=0.0 * total, + sources=0.0 * total, + reflux=0.0 * total, + projection=0.0 * total, + ) + route = ledger.route_identity(state.block) + balance_due_contract = BalanceDueContract( + make_identity("consumer-graph", {"test": "uniform-selective-replay"}), + ( + BalanceDueRoute( + route, + ( + BalanceDueConsumer( + make_identity( + "consumer-manifest", + {"test": "uniform-selective-replay"}, + ), + pops.time.every(2, clock=program.clock), + ), + ), + ), + ), + ) next_state = program.value( "Un", state.n @@ -93,7 +128,7 @@ def _program(model): ) program.commit(state.next, next_state) program.step_strategy(pops.time.FixedDt(DT_SEQUENCE[0])) - return program + return program, balance_due_contract def _initial_state(): @@ -183,8 +218,13 @@ def test_uniform_interval_history_variable_dt_restart_is_bit_identical(): model = passive_source_model( "uniform_selective_history_model", coefficient=COEFFICIENT ) - program = _program(model) - compiled = compile_problem(model=model, time=program, target="system") + program, balance_due_contract = _program(model) + compiled = compile_problem( + model=model, + time=program, + target="system", + balance_due_contract=balance_due_contract, + ) compiled_block = compile_block_model(model, target="system") initial = _initial_state() From 022da59fc607a57b692174e421daaf146d0f900f Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 01:29:18 +0200 Subject: [PATCH 035/109] fix(time): keep cadence in the core layer --- python/pops/runtime/_program_cadence_install.py | 2 +- python/pops/time/{_program/cadence.py => _cadence.py} | 2 +- python/pops/time/_graph/program.py | 2 +- python/pops/time/_program/api.py | 2 +- tests/python/unit/time/test_program_cadence.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) rename python/pops/time/{_program/cadence.py => _cadence.py} (95%) diff --git a/python/pops/runtime/_program_cadence_install.py b/python/pops/runtime/_program_cadence_install.py index 4c5b8511a..913caaaa9 100644 --- a/python/pops/runtime/_program_cadence_install.py +++ b/python/pops/runtime/_program_cadence_install.py @@ -6,7 +6,7 @@ def install_program_cadence(engine: Any, program: Any) -> None: """Install one authenticated cadence before the native Program and runtime freeze.""" - from pops.time._program.cadence import ProgramCadence + from pops.time._cadence import ProgramCadence from pops.time._program.contract import require_program require_program(program, exact=True, where="pops.bind Program cadence") diff --git a/python/pops/time/_program/cadence.py b/python/pops/time/_cadence.py similarity index 95% rename from python/pops/time/_program/cadence.py rename to python/pops/time/_cadence.py index ebea4a646..9a20d3329 100644 --- a/python/pops/time/_program/cadence.py +++ b/python/pops/time/_cadence.py @@ -1,4 +1,4 @@ -"""Immutable macro-step cadence authored by :class:`pops.time.Program`.""" +"""Immutable macro-step cadence shared by Program authoring and graph IR.""" from __future__ import annotations from dataclasses import dataclass diff --git a/python/pops/time/_graph/program.py b/python/pops/time/_graph/program.py index c5253ba02..1fc22f305 100644 --- a/python/pops/time/_graph/program.py +++ b/python/pops/time/_graph/program.py @@ -11,7 +11,7 @@ from pops.time._graph.nodes import NODE_TYPES from pops.time._graph.validation import validate_nodes from pops.time.points import Clock -from pops.time._program.cadence import ProgramCadence +from pops.time._cadence import ProgramCadence GRAPH_NODE_TYPES = (*NODE_TYPES, Branch, Loop) diff --git a/python/pops/time/_program/api.py b/python/pops/time/_program/api.py index 72b107213..b2398a645 100644 --- a/python/pops/time/_program/api.py +++ b/python/pops/time/_program/api.py @@ -13,7 +13,7 @@ from pops.model.ownership import OwnerKind, OwnerPath from pops.time._program.contract import register_program_type -from pops.time._program.cadence import ProgramCadence +from pops.time._cadence import ProgramCadence from pops.time._program.authoring import _ProgramAuthoring from pops.time._program.condensed import _ProgramCondensed from pops.time._program.operations import _ProgramCore diff --git a/tests/python/unit/time/test_program_cadence.py b/tests/python/unit/time/test_program_cadence.py index 726df656f..c2a9b07f1 100644 --- a/tests/python/unit/time/test_program_cadence.py +++ b/tests/python/unit/time/test_program_cadence.py @@ -6,7 +6,7 @@ from pops.identity.semantic import semantic_identity_of from pops.runtime._program_cadence_install import install_program_cadence from pops.time import Program -from pops.time._program.cadence import ProgramCadence +from pops.time._cadence import ProgramCadence from pops.time._program.detach import detach_compiled_program From c6e6f67335eae6b34fa714fc5ce51dc98853acd9 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 01:35:14 +0200 Subject: [PATCH 036/109] feat(output): detach async scientific diagnostics --- CHANGELOG.md | 5 + python/pops/output/_consumer_authoring.py | 10 +- python/pops/output/_consumer_contracts.py | 29 +- python/pops/output/observers.py | 46 +- python/pops/runtime/_runtime_consumers.py | 44 +- .../test_ci_impacted_selection.py | 10 +- .../mpi/test_async_balance_cadence_mpi.py | 422 ++++++++++++++++++ tests/python/test_durations.json | 8 +- ...est_async_scientific_output_diagnostics.py | 412 +++++++++++++++++ tests/test_manifest.toml | 1 + 10 files changed, 967 insertions(+), 20 deletions(-) create mode 100644 tests/python/integration/mpi/test_async_balance_cadence_mpi.py create mode 100644 tests/python/unit/output/test_async_scientific_output_diagnostics.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f8ee5fb1b..3d8962e64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,11 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning - `Program.cadence(substeps=..., stride=...)` now authors the native global cadence as immutable, identity-bearing Program data and installs it before the Uniform or AMR runtime freezes. +- `AsyncScientificOutput` now accepts fields, diagnostics, or both on one exact schedule. Diagnostic + reductions, including the five-term `Balance` ledger, are captured transactionally before the + accepted snapshot is detached; the asynchronous worker receives only immutable arrays and + scalars. Sparse Balance cadences elide off-cadence reductions, publish an exact zero ledger for + held Program strides, and replay accepted state without reopening the native mailbox. - AMR checkpoint capability reports now distinguish same-rank bit-identical replay from non-bit-identical rank-count rematerialization with Dense persisted histories, and state explicitly that `RegridOnRestart()` remains unsupported. The M3 gate now executes the persisted two-rank to diff --git a/python/pops/output/_consumer_authoring.py b/python/pops/output/_consumer_authoring.py index f2be575b2..ee84e6b1d 100644 --- a/python/pops/output/_consumer_authoring.py +++ b/python/pops/output/_consumer_authoring.py @@ -19,6 +19,7 @@ ParallelMode, _FAILURE_ACTIONS, _console_provider_data, + _is_async_scientific_observer, _observer_provider_data, ) @@ -119,6 +120,10 @@ def __post_init__(self) -> None: if operation_data["parallel_mode"] != self.parallel_mode.value: raise ValueError( "Monitor authoring parallel mode differs from its operation provider") + if rows and not _is_async_scientific_observer(operation_data): + raise ValueError( + "only AsyncScientificOutput monitor nodes can embed diagnostic providers" + ) elif self.kind is ConsumerKind.DIAGNOSTIC: if self.output_format is not None or self.operation is None: raise ValueError("Diagnostic authoring requires only its console provider") @@ -340,7 +345,10 @@ def resolve(self, resolver: Any, layout_plan: Any, *, owner: Any) -> ConsumerMan async_format.get("selection_contract") if isinstance(async_format, dict) else None ) - selected_layouts = {quantity.layout_id for quantity in quantities} + selected_layouts = { + quantity.layout_id + for quantity in (*quantities, *diagnostic_quantities) + } if isinstance(async_format, dict) and selection_contract is not None \ and selection_contract["layout_cardinality"] == "single" \ and len(selected_layouts) > 1: diff --git a/python/pops/output/_consumer_contracts.py b/python/pops/output/_consumer_contracts.py index 046896b35..6d4f7517c 100644 --- a/python/pops/output/_consumer_contracts.py +++ b/python/pops/output/_consumer_contracts.py @@ -136,6 +136,19 @@ def _observer_provider_data(value: Any, *, where: str) -> Mapping[str, Any]: return freeze_data(first, "%s.consumer_data" % where) +def _is_async_scientific_observer(operation_data: Any) -> bool: + """Authenticate the one monitor provider allowed to carry scientific diagnostics.""" + if not isinstance(operation_data, Mapping): + return False + observer = operation_data.get("observer") + return ( + isinstance(observer, Mapping) + and observer.get("observer_kind") == "async_scientific_output" + and observer.get("provider_id") + == "pops.output.async-scientific-writer.v1" + ) + + def _console_provider_data(value: Any, *, where: str) -> Mapping[str, Any]: """Authenticate the Python-only renderer of a rank-zero diagnostic consumer.""" if getattr(value, "__pops_ir_immutable__", False) is not True: @@ -553,10 +566,16 @@ def __post_init__(self) -> None: "descriptor": first, "references": [value.canonical_identity() for value in resolved_references], }, "%s.consumer_data" % where)) + async_scientific_monitor = ( + self.kind is ConsumerKind.MONITOR + and _is_async_scientific_observer(operation_data) + ) if diagnostic_rows and self.kind not in { - ConsumerKind.DIAGNOSTIC, ConsumerKind.SCIENTIFIC_OUTPUT}: + ConsumerKind.DIAGNOSTIC, ConsumerKind.SCIENTIFIC_OUTPUT + } and not async_scientific_monitor: raise ValueError( - "only ConsoleMonitor or ScientificOutput can embed diagnostic providers") + "only ConsoleMonitor, ScientificOutput, or AsyncScientificOutput " + "can embed diagnostic providers") object.__setattr__(self, "diagnostics_data", tuple(diagnostic_rows)) if not isinstance(self.diagnostic_quantities, tuple) or any( type(value) is not DiagnosticQuantity @@ -573,9 +592,11 @@ def __post_init__(self) -> None: raise ValueError( "ConsumerManifest must lower every diagnostic descriptor exactly once") if diagnostic_quantities and self.kind not in { - ConsumerKind.DIAGNOSTIC, ConsumerKind.SCIENTIFIC_OUTPUT}: + ConsumerKind.DIAGNOSTIC, ConsumerKind.SCIENTIFIC_OUTPUT + } and not async_scientific_monitor: raise ValueError( - "only ConsoleMonitor or ScientificOutput can carry diagnostic quantities") + "only ConsoleMonitor, ScientificOutput, or AsyncScientificOutput " + "can carry diagnostic quantities") has_accepted_balance = any( operation["reduction"] == "accepted_balance" for quantity in diagnostic_quantities diff --git a/python/pops/output/observers.py b/python/pops/output/observers.py index dafa47e80..afa82928d 100644 --- a/python/pops/output/observers.py +++ b/python/pops/output/observers.py @@ -1040,7 +1040,8 @@ class AsyncScientificOutput(Descriptor): SERIAL and gathered ROOT writers need no worker MPI. PER_RANK and COLLECTIVE writers execute on one duplicated MPI lane per consumer, isolated from numerical collectives. The default queue is process-lifetime only; a ``DurableJournal`` policy adds the explicit crash-replay - handoff. + handoff. Fields and diagnostic reductions share one exact schedule; diagnostics are reduced + before the immutable accepted snapshot is handed to the worker. """ category = "async_scientific_output" @@ -1050,7 +1051,8 @@ def __init__( *, format: Any, schedule: Any, - fields: Any, + fields: Any = (), + diagnostics: Any = (), levels: Any = None, target: Any, queue_capacity: Any = 1, @@ -1066,14 +1068,32 @@ def __init__( if type(schedule) is not Schedule: raise TypeError("AsyncScientificOutput.schedule must be an exact pops.time.Schedule") field_rows = tuple(fields) - if not field_rows: - raise ValueError("AsyncScientificOutput requires at least one field") if any(not isinstance(reference, Handle) for reference in field_rows): raise TypeError("AsyncScientificOutput fields must contain declaration Handles") if any(reference.kind not in _LIVE_FIELD_KINDS for reference in field_rows): raise TypeError("AsyncScientificOutput fields accept only state, field, or aux Handles") if len(set(field_rows)) != len(field_rows): raise ValueError("AsyncScientificOutput fields must be unique") + diagnostic_rows = tuple(diagnostics) + for index, diagnostic in enumerate(diagnostic_rows): + where = "AsyncScientificOutput diagnostics[%d]" % index + for method in ( + "declaration_references", + "resolve_references", + "consumer_data", + "freeze", + ): + if not callable(getattr(diagnostic, method, None)): + raise TypeError("%s must implement %s()" % (where, method)) + cadence = getattr(diagnostic, "cadence", None) + if cadence is not None and cadence != schedule: + raise ValueError( + "a diagnostic embedded in AsyncScientificOutput must use the same schedule" + ) + if not field_rows and not diagnostic_rows: + raise ValueError( + "AsyncScientificOutput requires at least one field or diagnostic" + ) selected_levels = AllLevels() if levels is None else levels if not isinstance(selected_levels, LevelSelection): raise TypeError("AsyncScientificOutput levels must be a typed LevelSelection") @@ -1098,6 +1118,7 @@ def __init__( self.format = format self.schedule = schedule self.fields = field_rows + self.diagnostics = diagnostic_rows self.levels = selected_levels self.target = _relative_target(target, where="AsyncScientificOutput.target") self.queue_capacity = queue_capacity @@ -1114,7 +1135,20 @@ def __init__( ) def declaration_references(self) -> tuple[Handle, ...]: - return self.fields + result = list(self.fields) + for index, diagnostic in enumerate(self.diagnostics): + references = diagnostic.declaration_references() + if not isinstance(references, tuple) or any( + not isinstance(reference, Handle) for reference in references + ): + raise TypeError( + "AsyncScientificOutput diagnostics[%d].declaration_references() " + "must return a tuple of Handles" % index + ) + for reference in references: + if reference not in result: + result.append(reference) + return tuple(result) def consumer_authoring(self) -> tuple[Any, ...]: from ._consumer_authoring import ConsumerAuthoringNode @@ -1130,6 +1164,7 @@ def consumer_authoring(self) -> tuple[Any, ...]: parallel_mode=self._operation.parallel_mode, levels=self.levels, operation=self._operation, + diagnostics=self.diagnostics, failure_action=FailRun(), ),) @@ -1138,6 +1173,7 @@ def options(self) -> dict[str, Any]: "format": self._operation.consumer_data()["observer"]["format"], "schedule": self.schedule.to_data(), "fields": [reference.inspect() for reference in self.fields], + "n_diagnostics": len(self.diagnostics), "levels": self.levels.to_data(), "target": self.target, "queue_capacity": self.queue_capacity, diff --git a/python/pops/runtime/_runtime_consumers.py b/python/pops/runtime/_runtime_consumers.py index 99c6d8f56..628137492 100644 --- a/python/pops/runtime/_runtime_consumers.py +++ b/python/pops/runtime/_runtime_consumers.py @@ -281,6 +281,10 @@ def effect_identity(self) -> Identity: def payload_identity(self) -> Identity: return self._effect.payload.identity + @property + def recoveries(self) -> tuple[Any, ...]: + return () + def publish(self) -> PublicationReceipt: if self._discarded: raise RuntimeError("discarded diagnostic cannot be published") @@ -464,7 +468,10 @@ def payload_identity(self) -> Identity: @property def recoveries(self) -> tuple[Any, ...]: - return self._output.recoveries + recoveries = getattr(self._output, "recoveries", ()) + if not isinstance(recoveries, tuple): + raise TypeError("prepared output recoveries must be a tuple") + return recoveries def publish(self) -> PublicationReceipt: if self._discarded: @@ -2832,13 +2839,27 @@ def publish_console( effect, values, publish_callback, self._discard_diagnostics, rollback ) + def _snapshot_for_effect( + self, + effect: AcceptedSideEffect, + manifest: Any, + ) -> tuple[OutputSnapshot, OutputRequest]: + if not getattr(manifest, "diagnostic_quantities", ()): + return self._owner._output_snapshot(manifest) + token = effect.identity.token + try: + diagnostics = self._pending[token] + except KeyError as error: + raise RuntimeError( + "scientific output diagnostics were not prepared for the accepted effect" + ) from error + return self._owner._output_snapshot(manifest, diagnostics) + def _resolve_output(self, effect: AcceptedSideEffect) -> OutputPreparation: manifest = self._manifest(effect) if manifest.output_format_data["provider_id"] == "pops.output.hdf5.v1": self._drain_post_commit_before_hdf5() - snapshot, request = self._owner._output_snapshot( - manifest, self._pending.get(effect.identity.token, ()) - ) + snapshot, request = self._snapshot_for_effect(effect, manifest) fmt = manifest.output_format format_name = manifest.output_format_data["format_name"] target = _target( @@ -2860,7 +2881,7 @@ def _prepare_live_visualization( effect: AcceptedSideEffect, manifest: Any, ) -> _PreparedLiveVisualization: - snapshot, request = self._owner._output_snapshot(manifest) + snapshot, request = self._snapshot_for_effect(effect, manifest) frame = None journal = None journal_record = None @@ -2920,7 +2941,18 @@ def prepare(self, effect: AcceptedSideEffect) -> PreparedPublication: if manifest.kind is ConsumerKind.DIAGNOSTIC: return self._prepare_diagnostic(effect, manifest) if manifest.kind is ConsumerKind.MONITOR: - return self._prepare_live_visualization(effect, manifest) + diagnostic = ( + self._prepare_diagnostic(effect, manifest) + if manifest.diagnostic_quantities + else None + ) + try: + live = self._prepare_live_visualization(effect, manifest) + except BaseException: + if diagnostic is not None: + diagnostic.discard() + raise + return live if diagnostic is None else _PreparedScientificOutput(live, diagnostic) if manifest.kind is ConsumerKind.SCIENTIFIC_OUTPUT: diagnostic = ( self._prepare_diagnostic(effect, manifest) diff --git a/tests/python/architecture/test_ci_impacted_selection.py b/tests/python/architecture/test_ci_impacted_selection.py index 087d2ad96..800898f08 100644 --- a/tests/python/architecture/test_ci_impacted_selection.py +++ b/tests/python/architecture/test_ci_impacted_selection.py @@ -614,6 +614,11 @@ def test_manifest_projects_exact_python_mpi_entrypoints(): "path": "tests/python/integration/mpi/test_amr_nonlinear_collective_mpi.py", "nproc": 2, }, + { + "suite": "pops_python_integration_mpi", + "path": "tests/python/integration/mpi/test_async_balance_cadence_mpi.py", + "nproc": 2, + }, { "suite": "pops_python_integration_mpi", "path": "tests/python/integration/mpi/test_scientific_output_mpi.py", @@ -667,6 +672,7 @@ class Args: "2\ttests/python/integration/mpi/test_amr_clean_route_program_mpi.py", "2\ttests/python/integration/mpi/test_amr_history_mpi.py", "2\ttests/python/integration/mpi/test_amr_nonlinear_collective_mpi.py", + "2\ttests/python/integration/mpi/test_async_balance_cadence_mpi.py", "2\ttests/python/integration/mpi/test_scientific_output_mpi.py", "2\ttests/python/integration/mpi/test_uniform_history_checkpoint_mpi.py", ] @@ -679,8 +685,8 @@ class Args: line.partition("=")[::2] for line in (tmp_path / "github-output.txt").read_text().splitlines() ) - assert outputs["python_mpi_count"] == "7" - assert outputs["python_mpi_entrypoint_count"] == "6" + assert outputs["python_mpi_count"] == "8" + assert outputs["python_mpi_entrypoint_count"] == "7" assert outputs["python_mpi_orchestrator_count"] == "1" diff --git a/tests/python/integration/mpi/test_async_balance_cadence_mpi.py b/tests/python/integration/mpi/test_async_balance_cadence_mpi.py new file mode 100644 index 000000000..40ce1a255 --- /dev/null +++ b/tests/python/integration/mpi/test_async_balance_cadence_mpi.py @@ -0,0 +1,422 @@ +#!/usr/bin/env python3 +"""Real MPI qualification of sparse Balance cadence and detached async snapshots. + +Both Uniform and two-level AMR execute the public +``Case -> Program.cadence -> compile -> mpi_world -> bind -> run`` route. The Program closes one +stride-3 window every third accepted macro-step, while async Balance consumers fire every two and +three accepted steps. Held windows must therefore publish exact zero ledgers and due windows must +publish native nonzero ledgers. A separate every-step async field series proves that each worker +receives the accepted field image captured on its own tick, never the latest native state. +""" +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from fractions import Fraction +from pathlib import Path +import os +import shutil +import tempfile +from typing import Any + +from _compile_once import compile_resolved_plan_once +from tests.python.support.requirements import require_mpi_or_skip + + +try: + import numpy as np + + import pops + from pops import _pops + from pops._native_collectives import ( + allgather_value, + barrier, + broadcast_value, + rank as world_rank, + size as world_size, + ) + from pops.amr import ( + AMRExecution, + AMRHierarchy, + AMRRegrid, + AMRTagging, + AMRTransfer, + Buffer, + ConflictPolicy, + EqualityPolicy, + Hysteresis, + PatchLayout, + Tag, + ) + from pops.codegen import Production + from pops.diagnostics import Balance, BalanceLedger + from pops.domain import Rectangle + from pops.frames import Cartesian2D + from pops.identity import make_identity + from pops.initial import InitialCondition + from pops.layouts import AMR, Uniform + from pops.lib.amr import StateTransfer + from pops.lib.initial import Gaussian + from pops.math import ValueExpr, ddt, div + from pops.mesh import CartesianGrid, PeriodicAxes + from pops.numerics import ( + DiscretizationPlan, + FiniteVolume, + reconstruction, + riemann, + variables, + ) + from pops.output import ( + AsyncScientificOutput, + ConsumerGraph, + NPZ, + ParallelMode, + read_npz, + ) + from pops.params import RuntimeParam + from pops.projection import ConservativeCellAverage + from pops.time import FixedDt, every +except Exception as exc: # noqa: BLE001 -- optional outside the required MPI lane + require_mpi_or_skip("async Balance MPI runtime import failed: %s" % exc) + + +ROOT = Path(__file__).resolve().parents[4] +N = 8 +DT = 1.0e-2 +NSTEPS = 6 +COMM = _pops.mpi_world() +RANK = world_rank(COMM) +SIZE = world_size(COMM) + + +if getattr(_pops, "__has_mpi__", False) is not True: + require_mpi_or_skip("async Balance cadence requires a native MPI build") +if SIZE != 2: + require_mpi_or_skip("async Balance cadence requires exactly mpiexec -n 2") + + +def _collective_local(label: str, operation: Any) -> Any: + result = None + error = None + try: + result = operation() + except BaseException as exc: # noqa: BLE001 -- publish every local cause before proceeding + error = "%s: %s" % (type(exc).__name__, exc) + errors = allgather_value(COMM, error) + failures = [ + "rank %d: %s" % (rank, value) + for rank, value in enumerate(errors) + if value is not None + ] + if failures: + raise RuntimeError("%s failed: %s" % (label, "; ".join(failures))) + return result + + +@contextmanager +def _shared_directory() -> Iterator[Path]: + local = tempfile.mkdtemp(prefix="pops-async-balance-mpi-") if RANK == 0 else None + root = Path(broadcast_value(COMM, local, root=0)) + barrier(COMM) + try: + yield root + finally: + barrier(COMM) + if RANK == 0: + shutil.rmtree(root, ignore_errors=True) + barrier(COMM) + + +def _authored_case(*, adaptive: bool) -> tuple[pops.Case, Any]: + label = "amr" if adaptive else "uniform" + frame = Rectangle( + "async-balance-%s-domain" % label, + lower=(0.0, 0.0), + upper=(1.0, 1.0), + ).frame(Cartesian2D()) + x_axis, y_axis = frame.axes + model = pops.Model("async-balance-%s-model" % label, frame=frame) + state = model.state("U", components=("rho",)) + (rho,) = state + flux = model.flux( + "zero_flux", + frame=frame, + state=state, + components={x_axis: (0.0 * rho,), y_axis: (0.0 * rho,)}, + waves={x_axis: (0.0 * rho,), y_axis: (0.0 * rho,)}, + ) + rate = model.rate("zero_rate", equation=ddt(state) == -div(flux)) + numerics = DiscretizationPlan() + numerics.rates.add( + rate, + FiniteVolume( + flux=flux, + variables=variables.Conservative(state), + reconstruction=reconstruction.FirstOrder(), + riemann=riemann.Rusanov(), + ), + ) + + case = pops.Case("async-balance-%s-case" % label) + block = case.block("tracer", model=model) + evolved = block[state] + case.numerics(numerics, block=block) + program = pops.Program("async-balance-%s-program" % label) + temporal = program.state(evolved) + total = program.sum(temporal.n) + zero = total * 0.0 + ledger = BalanceLedger("accepted-mass") + program.record_balance( + ledger, + storage_change=total, + outward_boundary_flux=zero, + sources=zero, + reflux=zero, + projection=zero, + ) + accepted = program.value( + "accepted_growth", + temporal.n + program.dt * Fraction(1, 2) * temporal.n, + at=temporal.next.point, + ) + program.commit(temporal.next, accepted) + program.cadence(stride=3) + program.step_strategy(FixedDt(DT)) + case.program(program) + + every_step = every(1, clock=program.clock) + every_two = every(2, clock=program.clock) + every_three = every(3, clock=program.clock) + root_npz = NPZ(mode=ParallelMode.ROOT) + case.consumers(ConsumerGraph.from_consumers(( + AsyncScientificOutput( + format=root_npz, + schedule=every_step, + fields=(evolved,), + target="state_every_1", + queue_capacity=1, + ), + AsyncScientificOutput( + format=root_npz, + schedule=every_two, + diagnostics=(Balance(ledger, block=block, cadence=every_two),), + target="balance_every_2", + queue_capacity=1, + ), + AsyncScientificOutput( + format=root_npz, + schedule=every_three, + diagnostics=(Balance(ledger, block=block, cadence=every_three),), + target="balance_every_3", + queue_capacity=1, + ), + ))) + case.initials.add(InitialCondition( + state=evolved, + value=Gaussian( + frame=frame, + center={x_axis: 0.5, y_axis: 0.5}, + background=1.0, + amplitude=0.5, + inverse_width=40.0, + ), + projection=ConservativeCellAverage(), + )) + grid = CartesianGrid( + frame=frame, + cells=(N, N), + periodic=PeriodicAxes(frame.axes), + ) + if not adaptive: + return case, Uniform(grid) + + threshold = case.param(RuntimeParam( + "async_balance_refine_threshold", + default=1.1, + )) + transfer = AMRTransfer() + transfer.state(evolved, StateTransfer()) + return case, AMR( + grid=grid, + hierarchy=AMRHierarchy(max_levels=2, ratios=(2,)), + tagging=AMRTagging( + rules=( + Tag(ValueExpr(evolved) > case.value(threshold)), + Buffer(cells=1), + ), + hysteresis=Hysteresis(0, EqualityPolicy.HOLD), + conflict_policy=ConflictPolicy.REFINE_WINS, + ), + regrid=AMRRegrid(schedule=every(100, clock=program.clock)), + transfer=transfer, + execution=AMRExecution.synchronous(), + patch_layout=PatchLayout( + distribute_coarse=True, + coarse_max_grid=4, + ), + ) + + +def _artifact(*, adaptive: bool) -> Any: + label = "amr" if adaptive else "uniform" + case, layout = _collective_local( + label + " authoring", + lambda: _authored_case(adaptive=adaptive), + ) + resolved = _collective_local( + label + " resolution", + lambda: pops.resolve( + pops.validate(case), + layout=layout, + backend=Production(), + compile_options={"include": str(ROOT / "include")}, + ), + ) + return compile_resolved_plan_once( + COMM, + resolved, + route="async-balance-" + label, + compile_artifact=pops.compile, + ) + + +def _snapshots(path: Path) -> dict[int, Any]: + result = {} + for artifact in path.rglob("*.npz"): + reopened = read_npz(artifact) + step = int(reopened.manifest["snapshot"]["clock"]["macro_step"]) + if step in result: + raise AssertionError("duplicate output at accepted step %d under %s" % (step, path)) + result[step] = reopened + return result + + +def _coarse_values(reopened: Any) -> np.ndarray: + snapshot = reopened.manifest["snapshot"] + field = next(row for row in snapshot["fields"] if row["key"]["level"] == 0) + token = make_identity("output-field", field["key"]).token + pieces = reopened.manifest["datasets"]["fields"][token]["pieces"] + return np.concatenate([ + np.asarray(reopened.arrays[piece["name"]]).ravel() + for piece in sorted(pieces, key=lambda row: (row["lower"], row["upper"])) + ]) + + +def _balance(reopened: Any) -> tuple[float, dict[str, float]]: + (payload,) = reopened.manifest["snapshot"]["diagnostics"] + return ( + float.fromhex(payload["value"]), + {name: float.fromhex(value) for name, value in payload["terms"].items()}, + ) + + +def _verify(root: Path, *, adaptive: bool) -> None: + if RANK != 0: + return + label = "amr" if adaptive else "uniform" + case_root = root / label + states = _snapshots(case_root / "state_every_1") + every_two = _snapshots(case_root / "balance_every_2") + every_three = _snapshots(case_root / "balance_every_3") + if set(states) != set(range(1, NSTEPS + 1)): + raise AssertionError("%s every-step async series is incomplete: %r" % (label, states)) + if set(every_two) != {2, 4, 6}: + raise AssertionError("%s every(2) Balance cadence differs: %r" % (label, every_two)) + if set(every_three) != {3, 6}: + raise AssertionError("%s every(3) Balance cadence differs: %r" % (label, every_three)) + + images = {step: _coarse_values(reopened) for step, reopened in states.items()} + if not np.array_equal(images[1], images[2]): + raise AssertionError("%s stride held state changed before step 3" % label) + if np.array_equal(images[2], images[3]): + raise AssertionError("%s due stride window did not advance at step 3" % label) + if not np.array_equal(images[3], images[4]) \ + or not np.array_equal(images[4], images[5]): + raise AssertionError("%s stride held state changed between steps 3 and 6" % label) + if np.array_equal(images[5], images[6]): + raise AssertionError("%s due stride window did not advance at step 6" % label) + + expected_terms = { + "storage_change", + "outward_boundary_flux", + "sources", + "reflux", + "projection", + } + for step in (2, 4): + value, terms = _balance(every_two[step]) + if value != 0.0 or set(terms) != expected_terms \ + or any(term != 0.0 for term in terms.values()): + raise AssertionError( + "%s held step %d did not publish the exact zero Balance ledger" + % (label, step) + ) + for series, steps in ((every_two, (6,)), (every_three, (3, 6))): + for step in steps: + value, terms = _balance(series[step]) + if set(terms) != expected_terms \ + or value <= 0.0 \ + or terms["storage_change"] <= 0.0 \ + or any( + terms[name] != 0.0 + for name in expected_terms - {"storage_change"} + ): + raise AssertionError( + "%s due step %d did not publish its native nonzero Balance ledger" + % (label, step) + ) + + +def _run_case(root: Path, *, adaptive: bool) -> None: + label = "amr" if adaptive else "uniform" + artifact = _artifact(adaptive=adaptive) + runtime = pops.bind( + artifact, + resources={"execution_context": pops.ExecutionContext.mpi_world(artifact)}, + ) + levels = allgather_value(COMM, int(runtime.n_levels())) + expected_levels = 2 if adaptive else 1 + if levels != (expected_levels,) * SIZE: + raise AssertionError("%s hierarchy differs across ranks: %r" % (label, levels)) + native_cadence = allgather_value( + COMM, + ( + int(runtime._executor._s.program_substeps()), + int(runtime._executor._s.program_stride()), + ), + ) + if native_cadence != ((1, 3),) * SIZE: + raise AssertionError( + "%s did not bind the public Program cadence: %r" % (label, native_cadence) + ) + report = pops.run( + runtime, + t_end=NSTEPS * DT, + max_steps=NSTEPS, + output_dir=root / label, + ) + reports = allgather_value( + COMM, + ( + report.accepted_steps, + report.run_identity.token, + report.bind_identity.token, + ), + ) + if any(row != reports[0] for row in reports[1:]) or reports[0][0] != NSTEPS: + raise AssertionError("%s run report differs across ranks: %r" % (label, reports)) + barrier(COMM) + _collective_local(label + " output verification", lambda: _verify(root, adaptive=adaptive)) + + +def main() -> None: + with _shared_directory() as root: + os.environ["POPS_CACHE_DIR"] = str(root / "cache") + _run_case(root, adaptive=False) + _run_case(root, adaptive=True) + if RANK == 0: + print("PASS test_async_balance_cadence_mpi") + + +if __name__ == "__main__": + main() diff --git a/tests/python/test_durations.json b/tests/python/test_durations.json index 05734f937..cd35ecff2 100644 --- a/tests/python/test_durations.json +++ b/tests/python/test_durations.json @@ -232,6 +232,7 @@ "tests/python/unit/numerics/test_finite_volume_composite.py": 1.0, "tests/python/unit/numerics/test_indicator_stencils.py": 1.0, "tests/python/unit/output/test_async_scientific_output.py": 2.0, + "tests/python/unit/output/test_async_scientific_output_diagnostics.py": 2.0, "tests/python/unit/output/test_durable_journal.py": 2.0, "tests/python/unit/output/test_durable_observer_integration.py": 2.0, "tests/python/unit/output/test_exact_writers.py": 1.0, @@ -361,6 +362,7 @@ "tests/python/unit/time/test_multirate_history_contract.py": 1.0, "tests/python/unit/time/test_operator_handle_resolution.py": 1.0, "tests/python/unit/time/test_program_authoring_atomicity.py": 1.0, + "tests/python/unit/time/test_program_cadence.py": 1.0, "tests/python/unit/time/test_program_deep_freeze.py": 1.0, "tests/python/unit/time/test_program_solve_final.py": 1.0, "tests/python/unit/time/test_program_to_graph.py": 1.0, @@ -407,7 +409,7 @@ "unit_seconds": "per-file pytest wall time", "measured_source": "borrowed _pops.so locally plus GitHub Actions run 30190778708 per-test timings", "estimated_note": "Unmeasured files use conservative path/content tiers (1/2/5/30/60/120 s); compiler-gated files retain native-compile estimates. Refresh every estimated row from a full CI run gate-python timing artifact.", - "estimated_count": 222, + "estimated_count": 224, "estimated_files": [ "tests/python/examples/final/test_hyqmom15_final_example.py", "tests/python/examples/final/test_scalar_advection_final_example.py", @@ -554,6 +556,7 @@ "tests/python/unit/numerics/test_finite_volume_composite.py", "tests/python/unit/numerics/test_indicator_stencils.py", "tests/python/unit/output/test_async_scientific_output.py", + "tests/python/unit/output/test_async_scientific_output_diagnostics.py", "tests/python/unit/output/test_durable_journal.py", "tests/python/unit/output/test_durable_observer_integration.py", "tests/python/unit/output/test_exact_writers.py", @@ -617,6 +620,7 @@ "tests/python/unit/time/test_multirate_history_contract.py", "tests/python/unit/time/test_operator_handle_resolution.py", "tests/python/unit/time/test_program_authoring_atomicity.py", + "tests/python/unit/time/test_program_cadence.py", "tests/python/unit/time/test_program_deep_freeze.py", "tests/python/unit/time/test_program_solve_final.py", "tests/python/unit/time/test_program_to_graph.py", @@ -632,6 +636,6 @@ "tests/python/unit/time/test_typed_provenance_guards.py", "tests/python/unit/time/test_typed_schedule.py" ], - "total_files": 403 + "total_files": 405 } } diff --git a/tests/python/unit/output/test_async_scientific_output_diagnostics.py b/tests/python/unit/output/test_async_scientific_output_diagnostics.py new file mode 100644 index 000000000..6a1298553 --- /dev/null +++ b/tests/python/unit/output/test_async_scientific_output_diagnostics.py @@ -0,0 +1,412 @@ +"""Diagnostics carried by AsyncScientificOutput are captured before post-commit dispatch.""" +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path +import threading + +import pytest + +from pops.codegen._compiled_artifact import CompiledSimulationArtifact +from pops.codegen._plans import BindInputs, InstallPlan +from pops.diagnostics import Balance, BalanceLedger, Integral +from pops.identity import Identity, make_identity +from pops.layouts import Uniform +from pops.mesh import normalize_layout_plan +from pops.model import Handle, OwnerKind, OwnerPath +from pops.output import ( + AsyncScientificOutput, + ConsumerGraph, + NPZ, + OutputPublicationReceipt, + ParallelMode, +) +from pops.output._consumer_authoring import ConsumerAuthoringNode +from pops.output._consumer_contracts import ( + ConsumerKind, + ConsumerManifest, + DiagnosticQuantity, +) +from pops.output._restart_provider import RestartAuthority +from pops.output._writers.common import writer_session_authority +from pops.problem.handles import BlockHandle +from pops.runtime._runtime_instance import RuntimeInstance +from pops.time import Clock, every +from tests.python.support.layout_plan import cartesian_grid +from tests.python.support.native_execution_context import artifact_execution_context +from tests.python.unit.runtime.test_consumer_authoring import _case +from tests.python.unit.runtime.test_runtime_instance_gate import ( + _Executor, + _install, + _scientific_output_mode, +) + + +def _resolved_async_balance(): + case, block, state = _case() + clock = Clock("async-balance", owner=case.owner_path) + schedule = every(2, clock=clock) + ledger = BalanceLedger("async-mass") + descriptor = AsyncScientificOutput( + format=NPZ(), + schedule=schedule, + diagnostics=(Balance(ledger, block=block, cadence=schedule),), + target="async/balance", + ) + graph = ConsumerGraph.from_consumers((descriptor,)) + case.consumers(graph) + import pops + + pops.validate(case) + subjects = case.layout_subjects() + layout = normalize_layout_plan( + Uniform(cartesian_grid(n=8)), + owner=case.owner_path.canonical(), + states=subjects.states, + fields=subjects.fields, + blocks=subjects.blocks, + handle_resolver=case.resolve, + ) + return ( + descriptor, + graph.resolve(case.resolve, layout, owner=case.owner_path.canonical()), + block, + case.resolve(block), + case.resolve(state), + schedule, + ledger, + ) + + +def test_async_scientific_output_accepts_diagnostic_only_and_resolves_balance(): + descriptor, graph, declared_block, block, state, schedule, ledger = ( + _resolved_async_balance() + ) + (manifest,) = graph.nodes + + assert descriptor.fields == () + assert descriptor.declaration_references() == (declared_block,) + assert manifest.kind is ConsumerKind.MONITOR + assert manifest.quantities == () + assert manifest.operation_data["observer"]["observer_kind"] == "async_scientific_output" + assert manifest.schedule == schedule + (quantity,) = manifest.diagnostic_quantities + assert quantity.reference == state + assert quantity.levels == (0,) + assert quantity.execution["operations"] == ( + { + "name": "balance", + "reduction": "accepted_balance", + "transform": "identity", + "metric_weighted": False, + "balance_route": ledger.route_identity(block).token, + }, + ) + + +def test_async_scientific_output_requires_a_field_or_diagnostic_and_matching_cadence(): + case, block, state = _case() + clock = Clock("async-validation", owner=case.owner_path) + schedule = every(2, clock=clock) + + with pytest.raises(ValueError, match="at least one field or diagnostic"): + AsyncScientificOutput( + format=NPZ(), + schedule=schedule, + target="async/empty", + ) + with pytest.raises(ValueError, match="must use the same schedule"): + AsyncScientificOutput( + format=NPZ(), + schedule=schedule, + diagnostics=( + Integral(block=block, cadence=every(3, clock=clock)), + ), + target="async/cadence-mismatch", + ) + + descriptor = AsyncScientificOutput( + format=NPZ(), + schedule=schedule, + fields=(state,), + diagnostics=(Integral(block=block, cadence=schedule),), + target="async/field-and-diagnostic", + ) + assert descriptor.declaration_references() == (state, block) + assert descriptor.options()["n_diagnostics"] == 1 + + +class _NonScientificObserver: + __pops_ir_immutable__ = True + + def consumer_data(self): + return { + "schema_version": 1, + "provider_id": "pops.test.forged-async-scientific-observer.v1", + "observer_kind": "async_scientific_output", + } + + def open_session(self, _execution_context): + raise AssertionError("authoring validation must not open an observer session") + + +def test_generic_monitor_cannot_smuggle_diagnostic_providers(): + from pops.output import AllLevels, LiveVisualization + + case, block, state = _case() + clock = Clock("generic-monitor", owner=case.owner_path) + schedule = every(1, clock=clock) + live = LiveVisualization( + observer=_NonScientificObserver(), + schedule=schedule, + fields=(state,), + ) + operation = live.consumer_authoring()[0].operation + + with pytest.raises(ValueError, match="only AsyncScientificOutput"): + ConsumerAuthoringNode( + label="invalid-monitor-diagnostic", + kind=ConsumerKind.MONITOR, + references=(state,), + schedule=schedule, + target_uri="live", + output_format=None, + parallel_mode=ParallelMode.SERIAL, + levels=AllLevels(), + operation=operation, + diagnostics=(Integral(block=block),), + ) + + _, graph, _, _, resolved_state, _, _ = _resolved_async_balance() + (valid_async_manifest,) = graph.nodes + forged_operation = LiveVisualization( + observer=_NonScientificObserver(), + schedule=valid_async_manifest.schedule, + fields=(resolved_state,), + ).consumer_authoring()[0].operation + with pytest.raises(ValueError, match="only ConsoleMonitor, ScientificOutput"): + replace(valid_async_manifest, operation=forged_operation) + + +class _CapturingWriterSession: + def __init__(self, owner, request, target: Path) -> None: + self.authority = writer_session_authority("capturing-async", request, target) + self.identity = Identity.from_token(self.authority["session_identity"]) + self._owner = owner + self._request = request + self._target = target + + def stage(self): + self._owner.writer_started.set() + if not self._owner.release_writer.wait(timeout=10): + raise TimeoutError("capturing async writer was not released") + + def abort_prepare(self): + return None + + def publish(self): + self._target.parent.mkdir(parents=True, exist_ok=True) + self._target.write_bytes(b"captured detached diagnostics\n") + return OutputPublicationReceipt( + self._target, + "capturing-async", + make_identity( + "scientific-output", + {"selection": self._request.publication_identity.token}, + ), + self._request.publication_identity, + ) + + def rollback(self): + self._target.unlink(missing_ok=True) + + def finalize(self): + return None + + +class _CapturingWriter: + format = "capturing-async" + + def __init__(self, owner) -> None: + self._owner = owner + + def preflight(self, _execution_context): + return {"schema_version": 1, "provider_id": "capturing-async"} + + def prepare_session(self, snapshot, request, target, *, communicator=None): + assert communicator is None + self._owner.worker_threads.append(threading.current_thread().name) + self._owner.snapshots.append(snapshot) + return _CapturingWriterSession(self._owner, request, Path(target)) + + +class _CapturingFormat: + __pops_ir_immutable__ = True + + def __init__(self, mode: ParallelMode) -> None: + self.mode = mode + self.writer_started = threading.Event() + self.release_writer = threading.Event() + self.worker_threads: list[str] = [] + self.snapshots = [] + + def consumer_data(self): + return { + "schema_version": 1, + "provider_id": "pops.test.capturing-async.v1", + "format_name": "capturing-async", + "extension": ".capture", + "parallel_mode": self.mode.value, + } + + def writer(self): + return _CapturingWriter(self) + + +class _BalanceExecutor(_Executor): + def __init__(self, plan): + super().__init__(plan) + self.mailbox_open = True + self.mailbox_calls: list[tuple[str, str]] = [] + + def _accepted_balance_terms(self, route): + if not self.mailbox_open: + raise RuntimeError("post-commit worker attempted to read the native balance mailbox") + self.mailbox_calls.append((threading.current_thread().name, route)) + return { + "storage_change": 7.0, + "outward_boundary_flux": 2.0, + "sources": 3.0, + "reflux": 1.0, + "projection": 0.5, + } + + +def _async_balance_runtime(tmp_path: Path): + base = _install() + mode = _scientific_output_mode(base.artifact) + layout = base.artifact.layout_plan.layouts[0] + block_subject = next( + assignment.subject + for assignment in base.artifact.layout_plan.assignments + if assignment.subject_kind == "block" + ) + block = BlockHandle( + block_subject.local_id, + owner=block_subject.owner_path, + model_owner=OwnerPath.model("adc-686-balance-fixture"), + ) + state = Handle( + "rho", + kind="state", + owner=block.owner_path.child(OwnerKind.BLOCK, block.local_id), + ) + clock = Clock("detached-async-balance", owner=OwnerPath.consumer("adc-686")) + schedule = every(1, clock=clock) + ledger = BalanceLedger("detached-async-balance") + balance = Balance(ledger, block=block, cadence=schedule) + format_provider = _CapturingFormat(mode) + descriptor = AsyncScientificOutput( + format=format_provider, + schedule=schedule, + diagnostics=(balance,), + target="detached-balance", + ) + node = descriptor.consumer_authoring()[0] + consumer = Handle("detached-balance", kind="consumer", owner=OwnerPath.consumer("adc-686")) + diagnostic = DiagnosticQuantity( + Handle( + "balance", + kind="diagnostic", + owner=consumer.owner_path.child( + OwnerKind.DESCRIPTOR, consumer.local_id + ).child(OwnerKind.DESCRIPTOR, "diagnostics"), + ), + state, + "state:fluid", + layout.handle.qualified_id, + (0,), + node.diagnostics[0].diagnostic_execution(), + ) + manifest = ConsumerManifest( + consumer, + ConsumerKind.MONITOR, + (), + schedule, + "detached-balance", + None, + mode, + operation=node.operation, + diagnostics=node.diagnostics, + diagnostic_quantities=(diagnostic,), + ) + graph = ConsumerGraph((manifest,)) + record = replace( + base.artifact.plan, + consumer_graph=graph, + restart_authority=RestartAuthority.from_consumer_graph(graph), + ) + artifact = CompiledSimulationArtifact( + record, + base.artifact.program, + base.artifact.blocks, + ) + inputs = BindInputs() + plan = InstallPlan( + artifact=artifact, + bind_inputs=inputs, + instances={ + installed.name: {"model": installed.model, "spatial": installed.spatial} + for installed in artifact.blocks + }, + params=artifact.bind_schema.resolve_bind( + {}, compile_values=artifact.plan.compile_values + ), + aux={}, + execution_context=artifact_execution_context(artifact), + ) + executor = _BalanceExecutor(plan) + runtime = RuntimeInstance(plan, executor=executor) + return runtime, executor, format_provider, manifest, ledger.route_identity(block).token + + +def test_async_worker_receives_detached_balance_payload_without_reopening_mailbox(tmp_path): + runtime, executor, format_provider, manifest, route = _async_balance_runtime(tmp_path) + reports = [] + failures = [] + + def run(): + try: + reports.append(runtime._run(t_end=1.0, max_steps=1, output_dir=tmp_path)) + except BaseException as error: # noqa: BLE001 - report worker/run failures together + failures.append(error) + + runner = threading.Thread(target=run, name="adc686-balance-runner", daemon=False) + runner.start() + assert format_provider.writer_started.wait(timeout=5) + assert executor.mailbox_calls == [("adc686-balance-runner", route)] + + executor.mailbox_open = False + format_provider.release_writer.set() + runner.join(timeout=10) + + assert not runner.is_alive() + assert failures == [] + assert len(reports) == 1 and reports[0].accepted_steps == 1 + assert len(format_provider.snapshots) == 1 + assert len(format_provider.worker_threads) == 1 + assert format_provider.worker_threads[0] != "adc686-balance-runner" + (payload,) = format_provider.snapshots[0].diagnostics + assert payload.value == pytest.approx(4.5) + assert dict(payload.terms) == { + "storage_change": 7.0, + "outward_boundary_flux": 2.0, + "sources": 3.0, + "reflux": 1.0, + "projection": 0.5, + } + assert executor.mailbox_calls == [("adc686-balance-runner", route)] + accepted = runtime.inspect().to_dict()["instance"]["accepted_diagnostics"] + assert len(accepted) == 1 + assert accepted[0]["value"] == (4.5).hex() + assert runtime.consumer_cursors.for_consumer(manifest.qualified_id).committed_samples == 1 diff --git a/tests/test_manifest.toml b/tests/test_manifest.toml index 4d99ca2ad..bfd20fe10 100644 --- a/tests/test_manifest.toml +++ b/tests/test_manifest.toml @@ -1180,6 +1180,7 @@ mpi_entrypoints = [ { path = "tests/python/integration/mpi/test_amr_clean_route_program_mpi.py", nproc = 2 }, { path = "tests/python/integration/mpi/test_amr_history_mpi.py", nproc = 2 }, { path = "tests/python/integration/mpi/test_amr_nonlinear_collective_mpi.py", nproc = 2 }, + { path = "tests/python/integration/mpi/test_async_balance_cadence_mpi.py", nproc = 2 }, { path = "tests/python/integration/mpi/test_scientific_output_mpi.py", nproc = 2 }, { path = "tests/python/integration/mpi/test_uniform_history_checkpoint_mpi.py", nproc = 2 }, ] From e5312deb9737fb522df2e375669822d8d32b2f22 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 01:35:48 +0200 Subject: [PATCH 037/109] fix(ci): isolate balance due contracts from output --- python/pops/_balance_due_contract.py | 210 ++++++++++++++++++++ python/pops/codegen/_compile_drivers.py | 2 +- python/pops/codegen/_phases.py | 2 +- python/pops/codegen/program_balance_due.py | 2 +- python/pops/codegen/program_codegen.py | 2 +- python/pops/codegen/program_emit_control.py | 2 +- python/pops/output/_balance_due_contract.py | 209 +------------------ 7 files changed, 221 insertions(+), 208 deletions(-) create mode 100644 python/pops/_balance_due_contract.py diff --git a/python/pops/_balance_due_contract.py b/python/pops/_balance_due_contract.py new file mode 100644 index 000000000..3052caaf7 --- /dev/null +++ b/python/pops/_balance_due_contract.py @@ -0,0 +1,210 @@ +"""Core typed bridge from one resolved ConsumerGraph to Balance producers.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from pops.identity import Identity, make_identity +from pops.time._schedule.api import Always, Every, Schedule, When +from pops.time._schedule.domains import AcceptedStep + + +_MAX_NATIVE_ACCEPTED_STEP = (1 << 31) - 1 + + +def _identity(value: Any, domain: str, *, where: str) -> Identity: + if type(value) is not Identity or value.domain != domain or value.schema_version != 1: + raise TypeError("%s must be an exact version-1 %s Identity" % (where, domain)) + return Identity.from_data(value.to_data()) + + +@dataclass(frozen=True, slots=True) +class BalanceDueConsumer: + """One exact ConsumerGraph node whose schedule requests a balance route.""" + + consumer: Identity + schedule: Schedule + + def __post_init__(self) -> None: + object.__setattr__( + self, + "consumer", + _identity( + self.consumer, + "consumer-manifest", + where="BalanceDueConsumer.consumer", + ), + ) + if type(self.schedule) is not Schedule: + raise TypeError("BalanceDueConsumer.schedule must be an exact Schedule") + + def to_data(self) -> dict[str, Any]: + return { + "consumer": self.consumer.to_data(), + "schedule": self.schedule.to_data(), + } + + +@dataclass(frozen=True, slots=True) +class BalanceDueRoute: + """All consumer schedules that request one exact native balance route.""" + + route: Identity + consumers: tuple[BalanceDueConsumer, ...] + + def __post_init__(self) -> None: + object.__setattr__( + self, + "route", + _identity( + self.route, + "balance-ledger-route", + where="BalanceDueRoute.route", + ), + ) + if not isinstance(self.consumers, tuple) or any( + type(value) is not BalanceDueConsumer for value in self.consumers + ): + raise TypeError( + "BalanceDueRoute.consumers must contain exact BalanceDueConsumer values" + ) + consumers = tuple( + sorted(self.consumers, key=lambda value: value.consumer.token) + ) + identities = [value.consumer.token for value in consumers] + if len(identities) != len(set(identities)): + raise ValueError("BalanceDueRoute contains a duplicate consumer") + object.__setattr__(self, "consumers", consumers) + + def to_data(self) -> dict[str, Any]: + return { + "route": self.route.to_data(), + "consumers": [value.to_data() for value in self.consumers], + } + + def accepted_step_periods(self) -> tuple[int, ...]: + """Return exact native periods, conservatively using period one when unprovable. + + ``Every(n)`` on the accepted-step domain is the first optimized cutover. ``Always`` and a + statically true ``When`` are exactly period one; a statically false ``When`` contributes no + occurrence. Any other domain/trigger remains active every step so this optimization can + never suppress evidence required by a ConsumerGraph extension or physical-time cadence. + """ + periods = [] + for row in self.consumers: + schedule = row.schedule + if type(schedule.domain) is not AcceptedStep: + return (1,) + trigger = schedule.trigger + if type(trigger) is Every: + # The native facade's public macro-step is a signed 32-bit ``int`` and rejects + # overflow before increment. A larger positive period can therefore never fire in + # any representable run; omit it instead of emitting an implementation-defined C++ + # narrowing conversion. + if trigger.n <= _MAX_NATIVE_ACCEPTED_STEP: + periods.append(trigger.n) + elif type(trigger) is Always: + periods.append(1) + elif type(trigger) is When and type(trigger.condition) is bool: + if trigger.condition: + periods.append(1) + else: + return (1,) + if 1 in periods: + return (1,) + return tuple(sorted(set(periods))) + + +@dataclass(frozen=True, slots=True) +class BalanceDueContract: + """Immutable ConsumerGraph-derived cadence authority consumed by native codegen.""" + + consumer_graph: Identity | None + routes: tuple[BalanceDueRoute, ...] + identity: Identity = field(init=False) + + def __post_init__(self) -> None: + if self.consumer_graph is not None: + object.__setattr__( + self, + "consumer_graph", + _identity( + self.consumer_graph, + "consumer-graph", + where="BalanceDueContract.consumer_graph", + ), + ) + if not isinstance(self.routes, tuple) or any( + type(value) is not BalanceDueRoute for value in self.routes + ): + raise TypeError( + "BalanceDueContract.routes must contain exact BalanceDueRoute values" + ) + routes = tuple(sorted(self.routes, key=lambda value: value.route.token)) + tokens = [value.route.token for value in routes] + if len(tokens) != len(set(tokens)): + raise ValueError("BalanceDueContract contains a duplicate route") + object.__setattr__(self, "routes", routes) + object.__setattr__( + self, + "identity", + make_identity("balance-due-contract", self._payload()), + ) + + @classmethod + def from_consumer_graph(cls, graph: Any) -> BalanceDueContract: + from pops.output._consumer_contracts import ConsumerGraph + + if graph is None: + return cls(None, ()) + if type(graph) is not ConsumerGraph or not graph.is_resolved: + raise TypeError( + "BalanceDueContract requires an exact resolved ConsumerGraph or None" + ) + by_route: dict[str, tuple[Identity, list[BalanceDueConsumer]]] = {} + for manifest in graph.nodes: + for quantity in manifest.diagnostic_quantities: + for operation in quantity.execution["operations"]: + if operation["reduction"] != "accepted_balance": + continue + route = Identity.from_token(operation["balance_route"]) + _identity( + route, + "balance-ledger-route", + where="accepted balance operation route", + ) + existing = by_route.setdefault(route.token, (route, [])) + existing[1].append( + BalanceDueConsumer(manifest.identity, manifest.schedule) + ) + return cls( + graph.identity, + tuple( + BalanceDueRoute(route, tuple(consumers)) + for route, consumers in by_route.values() + ), + ) + + def _payload(self) -> dict[str, Any]: + return { + "schema_version": 1, + "consumer_graph": ( + None if self.consumer_graph is None else self.consumer_graph.to_data() + ), + "routes": [value.to_data() for value in self.routes], + } + + def to_data(self) -> dict[str, Any]: + return {**self._payload(), "identity": self.identity.to_data()} + + def route(self, route: str) -> BalanceDueRoute | None: + if not isinstance(route, str) or not route: + raise TypeError("balance due route lookup requires non-empty text") + return next((value for value in self.routes if value.route.token == route), None) + + +__all__ = [ + "BalanceDueConsumer", + "BalanceDueContract", + "BalanceDueRoute", +] diff --git a/python/pops/codegen/_compile_drivers.py b/python/pops/codegen/_compile_drivers.py index 01ddb5997..674831d62 100644 --- a/python/pops/codegen/_compile_drivers.py +++ b/python/pops/codegen/_compile_drivers.py @@ -234,7 +234,7 @@ def compile_problem(so_path: Any = None, *, model: Any = None, model_graph: Any from pops.time._program.detach import detach_compiled_program time = detach_compiled_program(time) program_graph = time.to_graph() - from pops.output._balance_due_contract import BalanceDueContract + from pops._balance_due_contract import BalanceDueContract if balance_due_contract is None: balance_due_contract = BalanceDueContract.from_consumer_graph(None) if type(balance_due_contract) is not BalanceDueContract: diff --git a/python/pops/codegen/_phases.py b/python/pops/codegen/_phases.py index 9c1365344..1962606df 100644 --- a/python/pops/codegen/_phases.py +++ b/python/pops/codegen/_phases.py @@ -336,7 +336,7 @@ def compile(plan: Any) -> Any: from pops.codegen._compiled_artifact import CompiledLayoutProgram from pops.codegen.program_models import ProgramModelGraph from pops.codegen.program_balance_due import validate_balance_due_contract - from pops.output._balance_due_contract import BalanceDueContract + from pops._balance_due_contract import BalanceDueContract program = None options = dict(plan.compile_options) diff --git a/python/pops/codegen/program_balance_due.py b/python/pops/codegen/program_balance_due.py index 2459881bb..59c323b8a 100644 --- a/python/pops/codegen/program_balance_due.py +++ b/python/pops/codegen/program_balance_due.py @@ -9,7 +9,7 @@ from pops._balance_contract import BALANCE_TERM_NAMES from pops.identity import Identity -from pops.output._balance_due_contract import BalanceDueContract +from pops._balance_due_contract import BalanceDueContract from pops.time.values import ProgramValue diff --git a/python/pops/codegen/program_codegen.py b/python/pops/codegen/program_codegen.py index 3cca937e1..f3fd933f2 100644 --- a/python/pops/codegen/program_codegen.py +++ b/python/pops/codegen/program_codegen.py @@ -199,7 +199,7 @@ def emit_cpp_program( authority = model_graph if model_graph is not None else model if target not in ("system", "amr_system"): raise ValueError("emit_cpp_program: target 'system' | 'amr_system' (got %r)" % (target,)) - from pops.output._balance_due_contract import BalanceDueContract + from pops._balance_due_contract import BalanceDueContract if balance_due_contract is None: balance_due_contract = BalanceDueContract.from_consumer_graph(None) if type(balance_due_contract) is not BalanceDueContract: diff --git a/python/pops/codegen/program_emit_control.py b/python/pops/codegen/program_emit_control.py index 30e0ee0b8..8dfae01e5 100644 --- a/python/pops/codegen/program_emit_control.py +++ b/python/pops/codegen/program_emit_control.py @@ -248,7 +248,7 @@ def _emit_body(program: Any, model: Any = None, target: Any = "system", prepare_balance_due_lowering, ) if balance_due_contract is None: - from pops.output._balance_due_contract import BalanceDueContract + from pops._balance_due_contract import BalanceDueContract balance_due_contract = BalanceDueContract.from_consumer_graph(None) emit_balance_due_guards( prepare_balance_due_lowering(program, balance_due_contract), diff --git a/python/pops/output/_balance_due_contract.py b/python/pops/output/_balance_due_contract.py index 25919fb86..847d7fbe2 100644 --- a/python/pops/output/_balance_due_contract.py +++ b/python/pops/output/_balance_due_contract.py @@ -1,207 +1,10 @@ -"""Typed compile-time bridge from one resolved ConsumerGraph to Balance producers.""" -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import Any - -from pops.identity import Identity, make_identity -from pops.time._schedule.api import Always, Every, Schedule, When -from pops.time._schedule.domains import AcceptedStep - - -_MAX_NATIVE_ACCEPTED_STEP = (1 << 31) - 1 - - -def _identity(value: Any, domain: str, *, where: str) -> Identity: - if type(value) is not Identity or value.domain != domain or value.schema_version != 1: - raise TypeError("%s must be an exact version-1 %s Identity" % (where, domain)) - return Identity.from_data(value.to_data()) - - -@dataclass(frozen=True, slots=True) -class BalanceDueConsumer: - """One exact ConsumerGraph node whose schedule requests a balance route.""" - - consumer: Identity - schedule: Schedule - - def __post_init__(self) -> None: - object.__setattr__( - self, - "consumer", - _identity( - self.consumer, - "consumer-manifest", - where="BalanceDueConsumer.consumer", - ), - ) - if type(self.schedule) is not Schedule: - raise TypeError("BalanceDueConsumer.schedule must be an exact Schedule") - - def to_data(self) -> dict[str, Any]: - return { - "consumer": self.consumer.to_data(), - "schedule": self.schedule.to_data(), - } - - -@dataclass(frozen=True, slots=True) -class BalanceDueRoute: - """All consumer schedules that request one exact native balance route.""" - - route: Identity - consumers: tuple[BalanceDueConsumer, ...] - - def __post_init__(self) -> None: - object.__setattr__( - self, - "route", - _identity( - self.route, - "balance-ledger-route", - where="BalanceDueRoute.route", - ), - ) - if not isinstance(self.consumers, tuple) or any( - type(value) is not BalanceDueConsumer for value in self.consumers - ): - raise TypeError( - "BalanceDueRoute.consumers must contain exact BalanceDueConsumer values" - ) - consumers = tuple( - sorted(self.consumers, key=lambda value: value.consumer.token) - ) - identities = [value.consumer.token for value in consumers] - if len(identities) != len(set(identities)): - raise ValueError("BalanceDueRoute contains a duplicate consumer") - object.__setattr__(self, "consumers", consumers) - - def to_data(self) -> dict[str, Any]: - return { - "route": self.route.to_data(), - "consumers": [value.to_data() for value in self.consumers], - } - - def accepted_step_periods(self) -> tuple[int, ...]: - """Return exact native periods, conservatively using period one when unprovable. - - ``Every(n)`` on the accepted-step domain is the first optimized cutover. ``Always`` and a - statically true ``When`` are exactly period one; a statically false ``When`` contributes no - occurrence. Any other domain/trigger remains active every step so this optimization can - never suppress evidence required by a ConsumerGraph extension or physical-time cadence. - """ - periods = [] - for row in self.consumers: - schedule = row.schedule - if type(schedule.domain) is not AcceptedStep: - return (1,) - trigger = schedule.trigger - if type(trigger) is Every: - # The native facade's public macro-step is a signed 32-bit ``int`` and rejects - # overflow before increment. A larger positive period can therefore never fire in - # any representable run; omit it instead of emitting an implementation-defined C++ - # narrowing conversion. - if trigger.n <= _MAX_NATIVE_ACCEPTED_STEP: - periods.append(trigger.n) - elif type(trigger) is Always: - periods.append(1) - elif type(trigger) is When and type(trigger.condition) is bool: - if trigger.condition: - periods.append(1) - else: - return (1,) - if 1 in periods: - return (1,) - return tuple(sorted(set(periods))) - - -@dataclass(frozen=True, slots=True) -class BalanceDueContract: - """Immutable ConsumerGraph-derived cadence authority consumed by native codegen.""" - - consumer_graph: Identity | None - routes: tuple[BalanceDueRoute, ...] - identity: Identity = field(init=False) - - def __post_init__(self) -> None: - if self.consumer_graph is not None: - object.__setattr__( - self, - "consumer_graph", - _identity( - self.consumer_graph, - "consumer-graph", - where="BalanceDueContract.consumer_graph", - ), - ) - if not isinstance(self.routes, tuple) or any( - type(value) is not BalanceDueRoute for value in self.routes - ): - raise TypeError( - "BalanceDueContract.routes must contain exact BalanceDueRoute values" - ) - routes = tuple(sorted(self.routes, key=lambda value: value.route.token)) - tokens = [value.route.token for value in routes] - if len(tokens) != len(set(tokens)): - raise ValueError("BalanceDueContract contains a duplicate route") - object.__setattr__(self, "routes", routes) - object.__setattr__( - self, - "identity", - make_identity("balance-due-contract", self._payload()), - ) - - @classmethod - def from_consumer_graph(cls, graph: Any) -> BalanceDueContract: - from pops.output._consumer_contracts import ConsumerGraph - - if graph is None: - return cls(None, ()) - if type(graph) is not ConsumerGraph or not graph.is_resolved: - raise TypeError( - "BalanceDueContract requires an exact resolved ConsumerGraph or None" - ) - by_route: dict[str, tuple[Identity, list[BalanceDueConsumer]]] = {} - for manifest in graph.nodes: - for quantity in manifest.diagnostic_quantities: - for operation in quantity.execution["operations"]: - if operation["reduction"] != "accepted_balance": - continue - route = Identity.from_token(operation["balance_route"]) - _identity( - route, - "balance-ledger-route", - where="accepted balance operation route", - ) - existing = by_route.setdefault(route.token, (route, [])) - existing[1].append( - BalanceDueConsumer(manifest.identity, manifest.schedule) - ) - return cls( - graph.identity, - tuple( - BalanceDueRoute(route, tuple(consumers)) - for route, consumers in by_route.values() - ), - ) - - def _payload(self) -> dict[str, Any]: - return { - "schema_version": 1, - "consumer_graph": ( - None if self.consumer_graph is None else self.consumer_graph.to_data() - ), - "routes": [value.to_data() for value in self.routes], - } - - def to_data(self) -> dict[str, Any]: - return {**self._payload(), "identity": self.identity.to_data()} - - def route(self, route: str) -> BalanceDueRoute | None: - if not isinstance(route, str) or not route: - raise TypeError("balance due route lookup requires non-empty text") - return next((value for value in self.routes if value.route.token == route), None) +"""Compatibility aliases for the core balance due contract.""" +from pops._balance_due_contract import ( + BalanceDueConsumer, + BalanceDueContract, + BalanceDueRoute, +) __all__ = [ "BalanceDueConsumer", From 5135fd20a0d1bda062a5ef8069a986c2174e79de Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 01:35:55 +0200 Subject: [PATCH 038/109] docs(output): specify cadence and async balance semantics --- docs/design/exact-output-consumers.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/docs/design/exact-output-consumers.md b/docs/design/exact-output-consumers.md index 89046b252..33b265125 100644 --- a/docs/design/exact-output-consumers.md +++ b/docs/design/exact-output-consumers.md @@ -446,6 +446,11 @@ ScientificOutput( ) ``` +`AsyncScientificOutput(..., diagnostics=(Balance(mass, block=fluid),))` uses the same exact +schedule and transaction. Its reductions are completed on the simulation thread before detachment; +the post-commit worker receives only immutable arrays and scalar payloads, never the native mailbox +or communicator facade. + Each argument to `record_balance` is a signed, time-integrated native Program sum/dot reduction, or scalar arithmetic composed only from such reductions and exact literals. The reported residual is `storage_change + outward_boundary_flux - sources - reflux - projection`. @@ -474,13 +479,24 @@ operation, that shared producer remains unconditional so cadence fusion cannot c semantics. A `Balance` consumer with no matching five-term `Program.record_balance` producer fails before native code generation. Program stride/substeps use one attempt-local outer accepted-step target, so every substep of one due public step sees the same decision and accumulates into the same -attempt mailbox. +attempt mailbox. The cadence is authored once as part of the Program identity, for example +`program.cadence(substeps=2, stride=3)`, then authenticated and installed before runtime freeze on +both Uniform and AMR targets. A stride-held public step executes no Program work and therefore +publishes the exact additive-identity balance (all five terms are zero); a due Program that omits +even one term still fails closed. Accepted-step periods larger than the native signed-32-bit ceiling +can never fire in a representable run and are compiled off instead of being narrowed into C++. +Selective checkpoint reconstruction may re-execute the Program to rebuild omitted history slots, +but that work is not a public accepted step. Uniform and AMR replay therefore enter an explicit +native replay guard: every Balance due query returns false, no term reaches the accepted-attempt +mailbox, and the guard is restored on both success and exception. The replay still executes all +non-Balance scientific operations needed to reconstruct the history exactly. This first sparse cutover is exact only for accepted-step `every(n)` schedules. Physical-time `every_dt`, `on_end`, and extension domains/triggers remain conservatively active for every Program invocation; their consumer still publishes only when its own runtime schedule is due, but upstream balance reductions are not yet skipped. This fallback can add work but cannot suppress required -evidence. +evidence. A zero-step run has no accepted native occurrence: its coincident start/end moment cannot +publish an accepted-step consumer, including `Balance`. This route is explicit evidence, not automatic numerical instrumentation: a Program that cannot produce its actual reflux or projection increment cannot declare `Balance`. In particular, the From e656950b8e62719ff489b908ae8c892d08280b50 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 02:01:32 +0200 Subject: [PATCH 039/109] fix(ci): reconcile merged Python duration catalog --- tests/python/test_durations.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python/test_durations.json b/tests/python/test_durations.json index 773fbf903..0e967363f 100644 --- a/tests/python/test_durations.json +++ b/tests/python/test_durations.json @@ -410,7 +410,7 @@ "unit_seconds": "per-file pytest wall time", "measured_source": "borrowed _pops.so locally plus GitHub Actions run 30190778708 per-test timings", "estimated_note": "Unmeasured files use conservative path/content tiers (1/2/5/30/60/120 s); compiler-gated files retain native-compile estimates. Refresh every estimated row from a full CI run gate-python timing artifact.", - "estimated_count": 224, + "estimated_count": 225, "estimated_files": [ "tests/python/examples/final/test_hyqmom15_final_example.py", "tests/python/examples/final/test_scalar_advection_final_example.py", @@ -638,6 +638,6 @@ "tests/python/unit/time/test_typed_provenance_guards.py", "tests/python/unit/time/test_typed_schedule.py" ], - "total_files": 405 + "total_files": 406 } } From 63959ca5bdcb4a7dd56e5f6cb5ac86db380fedbc Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 02:11:19 +0200 Subject: [PATCH 040/109] fix(output): keep root async writers nonblocking --- python/pops/runtime/_runtime_consumers.py | 8 ++++++-- .../unit/runtime/test_runtime_instance_gate.py | 18 ++++++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/python/pops/runtime/_runtime_consumers.py b/python/pops/runtime/_runtime_consumers.py index 628137492..a0d040ee1 100644 --- a/python/pops/runtime/_runtime_consumers.py +++ b/python/pops/runtime/_runtime_consumers.py @@ -2052,13 +2052,17 @@ def _submit_live_visualization( raise RuntimeError("post-commit consensus accepted no exact run identity") if submission is not None: submission.arm() - if manifest.parallel_mode is not ParallelMode.SERIAL: + if manifest.parallel_mode in ( + ParallelMode.PER_RANK, + ParallelMode.COLLECTIVE, + ): # A Catalyst implementation may enter MPI from its worker thread even when PoPS gives # it a duplicated communicator. Do not let the next AMR/native step concurrently # enter solver collectives on the main thread: MPICH and third-party VTK internals do # not guarantee progress for that cross-library ordering. Drain the accepted live # frame locally, then prove every rank has left the worker lane before any rank returns - # to the solver. Serial observers and asynchronous scientific writers remain async. + # to the solver. SERIAL and gathered ROOT workers never enter MPI, so they remain + # asynchronous with the next numerical step. delivery_error = None try: self._observer_queue(manifest, run_identity).flush() diff --git a/tests/python/unit/runtime/test_runtime_instance_gate.py b/tests/python/unit/runtime/test_runtime_instance_gate.py index e26cea043..5ab7a73ae 100644 --- a/tests/python/unit/runtime/test_runtime_instance_gate.py +++ b/tests/python/unit/runtime/test_runtime_instance_gate.py @@ -388,6 +388,15 @@ def _with_graph( "state:u", layout.qualified_id, ) + resolved_mode = ( + parallel_mode + if kind is ConsumerKind.SCIENTIFIC_OUTPUT + else ( + ParallelMode(operation.consumer_data()["parallel_mode"]) + if kind is ConsumerKind.MONITOR + else ParallelMode.SERIAL + ) + ) manifest = ConsumerManifest( Handle("density", kind="consumer", owner=OwnerPath.consumer("adc-687")), kind, @@ -397,7 +406,7 @@ def _with_graph( NPZ(mode=parallel_mode) if output_format is None and kind is ConsumerKind.SCIENTIFIC_OUTPUT else output_format, - parallel_mode if kind is ConsumerKind.SCIENTIFIC_OUTPUT else ParallelMode.SERIAL, + resolved_mode, operation=operation, ) graph = ConsumerGraph((manifest,)) @@ -710,7 +719,8 @@ def prepare_session(self, snapshot, request, target, *, communicator=None): class _BlockingFormat: __pops_ir_immutable__ = True - def __init__(self): + def __init__(self, mode: ParallelMode): + self._mode = mode self.writer_started = threading.Event() self.release_writer = threading.Event() self.paths = [] @@ -721,7 +731,7 @@ def consumer_data(self): "provider_id": "pops.test.blocking-async.v1", "format_name": "blocking-test", "extension": ".async", - "parallel_mode": "serial", + "parallel_mode": self._mode.value, } def writer(self): @@ -731,7 +741,7 @@ def writer(self): def test_async_scientific_output_overlaps_next_step_and_flushes_real_receipts(tmp_path): output_root = tmp_path / "async-output" output_root.mkdir() - format_provider = _BlockingFormat() + format_provider = _BlockingFormat(_scientific_output_mode(_install().artifact)) authoring_clock = Clock("async-authoring") descriptor = AsyncScientificOutput( format=format_provider, From 8bb4d2a964093a45137a9a55f8674e8445dc93b9 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 02:31:24 +0200 Subject: [PATCH 041/109] fix(runtime): type authenticated cadence callables --- python/pops/runtime/_program_cadence_install.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/python/pops/runtime/_program_cadence_install.py b/python/pops/runtime/_program_cadence_install.py index 913caaaa9..9d1cbef52 100644 --- a/python/pops/runtime/_program_cadence_install.py +++ b/python/pops/runtime/_program_cadence_install.py @@ -1,7 +1,8 @@ """Bind-time installation of the immutable cadence carried by a compiled Program.""" from __future__ import annotations -from typing import Any +from collections.abc import Callable +from typing import Any, cast def install_program_cadence(engine: Any, program: Any) -> None: @@ -27,7 +28,9 @@ def install_program_cadence(engine: Any, program: Any) -> None: stride = getattr(engine, "program_stride", None) if not callable(substeps) or not callable(stride): raise RuntimeError("pops.bind runtime cannot authenticate the installed Program cadence") - actual = (int(substeps()), int(stride())) + installed_substeps = cast(Callable[[], int], substeps) + installed_stride = cast(Callable[[], int], stride) + actual = (int(installed_substeps()), int(installed_stride())) expected = (contract.substeps, contract.stride) if actual != expected: raise RuntimeError( From fb02634689d99f0365674f880cfa6d8866bbcc36 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 03:59:59 +0200 Subject: [PATCH 042/109] test(mpi): prove exact nonzero balance terms --- .../mpi/test_async_balance_cadence_mpi.py | 98 ++++++++++++++----- 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/tests/python/integration/mpi/test_async_balance_cadence_mpi.py b/tests/python/integration/mpi/test_async_balance_cadence_mpi.py index 40ce1a255..0ba380290 100644 --- a/tests/python/integration/mpi/test_async_balance_cadence_mpi.py +++ b/tests/python/integration/mpi/test_async_balance_cadence_mpi.py @@ -5,8 +5,11 @@ ``Case -> Program.cadence -> compile -> mpi_world -> bind -> run`` route. The Program closes one stride-3 window every third accepted macro-step, while async Balance consumers fire every two and three accepted steps. Held windows must therefore publish exact zero ledgers and due windows must -publish native nonzero ledgers. A separate every-step async field series proves that each worker -receives the accepted field image captured on its own tick, never the latest native state. +publish an exact signed five-term ledger built from five real collective Program reductions. The +fixture explicitly authors that accounting split; it proves transport, signs, residual closure and +rank agreement, not automatic extraction of AMR reflux or projection terms. A separate every-step +async field series proves that each worker receives the accepted field image captured on its own +tick, never the latest native state. """ from __future__ import annotations @@ -163,22 +166,35 @@ def _authored_case(*, adaptive: bool) -> tuple[pops.Case, Any]: case.numerics(numerics, block=block) program = pops.Program("async-balance-%s-program" % label) temporal = program.state(evolved) - total = program.sum(temporal.n) - zero = total * 0.0 - ledger = BalanceLedger("accepted-mass") - program.record_balance( - ledger, - storage_change=total, - outward_boundary_flux=zero, - sources=zero, - reflux=zero, - projection=zero, - ) accepted = program.value( "accepted_growth", temporal.n + program.dt * Fraction(1, 2) * temporal.n, at=temporal.next.point, ) + increment = program.value( + "accepted_increment", + accepted - temporal.n, + at=temporal.next.point, + ) + # This is an explicitly authored accounting fixture, not an automatic AMR-term extractor. + # Every term owns a real native Program.sum so the installed mpiexec route enters five + # collectives. The signed split closes the actual accepted storage increment exactly: + # storage + outward - sources - reflux - projection + # = q - q - q - q - (-2q) = 0. + storage_change = program.sum(increment) + outward_boundary_flux = -program.sum(increment) + sources = program.sum(increment) + reflux = program.sum(increment) + projection = -2.0 * program.sum(increment) + ledger = BalanceLedger("accepted-mass") + program.record_balance( + ledger, + storage_change=storage_change, + outward_boundary_flux=outward_boundary_flux, + sources=sources, + reflux=reflux, + projection=projection, + ) program.commit(temporal.next, accepted) program.cadence(stride=3) program.step_strategy(FixedDt(DT)) @@ -310,6 +326,34 @@ def _balance(reopened: Any) -> tuple[float, dict[str, float]]: ) +def _require_exact_signed_balance( + label: str, + step: int, + value: float, + terms: dict[str, float], +) -> None: + q = terms["storage_change"] + expected = { + "storage_change": q, + "outward_boundary_flux": -q, + "sources": q, + "reflux": q, + "projection": -2.0 * q, + } + residual = ( + terms["storage_change"] + + terms["outward_boundary_flux"] + - terms["sources"] + - terms["reflux"] + - terms["projection"] + ) + if q <= 0.0 or terms != expected or residual != 0.0 or value != residual: + raise AssertionError( + "%s due step %d did not preserve its exact signed five-term Balance: " + "value=%r terms=%r" % (label, step, value, terms) + ) + + def _verify(root: Path, *, adaptive: bool) -> None: if RANK != 0: return @@ -354,17 +398,13 @@ def _verify(root: Path, *, adaptive: bool) -> None: for series, steps in ((every_two, (6,)), (every_three, (3, 6))): for step in steps: value, terms = _balance(series[step]) - if set(terms) != expected_terms \ - or value <= 0.0 \ - or terms["storage_change"] <= 0.0 \ - or any( - terms[name] != 0.0 - for name in expected_terms - {"storage_change"} - ): - raise AssertionError( - "%s due step %d did not publish its native nonzero Balance ledger" - % (label, step) - ) + if set(terms) != expected_terms: + raise AssertionError("%s due step %d omitted a Balance term" % (label, step)) + _require_exact_signed_balance(label, step, value, terms) + if _balance(every_two[6]) != _balance(every_three[6]): + raise AssertionError( + "%s independent due consumers disagreed on the accepted step-6 Balance" % label + ) def _run_case(root: Path, *, adaptive: bool) -> None: @@ -405,6 +445,16 @@ def _run_case(root: Path, *, adaptive: bool) -> None: ) if any(row != reports[0] for row in reports[1:]) or reports[0][0] != NSTEPS: raise AssertionError("%s run report differs across ranks: %r" % (label, reports)) + accepted_balance = tuple( + row + for row in runtime.inspect().to_dict()["instance"]["accepted_diagnostics"] + if row["key"]["reduction"] == "discrete_balance" + ) + accepted_by_rank = allgather_value(COMM, accepted_balance) + if not accepted_balance or any(row != accepted_by_rank[0] for row in accepted_by_rank[1:]): + raise AssertionError( + "%s accepted Balance registry differs across ranks: %r" % (label, accepted_by_rank) + ) barrier(COMM) _collective_local(label + " output verification", lambda: _verify(root, adaptive=adaptive)) From 0676a12f2817e1939455b5a115633c3cabfdf7e6 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:49:16 +0200 Subject: [PATCH 043/109] feat(runtime): enforce planned determinism at install --- .../runtime_instance_planning_contract.md | 10 +-- python/pops/runtime/_runtime_executor.py | 46 +++++++++++++- .../runtime/test_runtime_executor_context.py | 63 +++++++++++++++++++ 3 files changed, 112 insertions(+), 7 deletions(-) diff --git a/docs/design/runtime_instance_planning_contract.md b/docs/design/runtime_instance_planning_contract.md index 34aadbf35..5b0d660a2 100644 --- a/docs/design/runtime_instance_planning_contract.md +++ b/docs/design/runtime_instance_planning_contract.md @@ -77,10 +77,12 @@ component reduction; adaptive integrals use the native volume-weighted composite exact selected levels. Installation and execution must authenticate the bundle's plan, bind, component and layout identities without rebuilding or weakening them. Every native provider authenticates the exact bundle before reading backend state or constructing -an execution engine; a missing or mismatched bundle therefore fails before execution. The complete -bundle is retained in the array-free `RuntimeInstance.inspect()` report under `instance.runtime_plan` -so derived halos, transfers, collectives, fences, buffers and determinism assumptions remain -reviewable rather than becoming hidden installation state. +an execution engine, then checks the plan's determinism guarantee against current native +rank/device/backend facts and its authenticated reduction order before native preflight; a missing +bundle, mismatched authority or changed execution fact therefore fails before execution. The +complete bundle is retained in the array-free `RuntimeInstance.inspect()` report under +`instance.runtime_plan` so derived halos, transfers, collectives, fences, buffers and determinism +assumptions remain reviewable rather than becoming hidden installation state. For an accepted step, successful native finalization is an irreversible `native_finalized` boundary. The instance commits the engine state, accepted cursor set and consumer receipts across diff --git a/python/pops/runtime/_runtime_executor.py b/python/pops/runtime/_runtime_executor.py index 53ef2bf4b..9db79f01e 100644 --- a/python/pops/runtime/_runtime_executor.py +++ b/python/pops/runtime/_runtime_executor.py @@ -78,7 +78,9 @@ def _uniform_initial_sources(plan: Any) -> dict[str, dict[str, Any]]: return result -def _require_supported_execution_context(plan: Any) -> None: +def _require_supported_execution_context( + plan: Any, native_facts: dict[str, Any] | None = None +) -> None: """Refuse every resource the native engines cannot consume before constructing one.""" from pops._platform_contracts import ExecutionContext @@ -89,7 +91,7 @@ def _require_supported_execution_context(plan: Any) -> None: raise NotImplementedError( "native RuntimeInstance providers require exact float64" ) - facts = _native_runtime_facts() + facts = _native_runtime_facts() if native_facts is None else native_facts expected_device = facts.get("kokkos_device") expected_memory = facts.get("field_memory_space") expected_backend = facts.get("kokkos_backend") @@ -150,6 +152,42 @@ def _require_supported_execution_context(plan: Any) -> None: ) +def _require_runtime_determinism( + plan: Any, runtime_plan: Any, native_facts: dict[str, Any] +) -> None: + """Consume the plan's determinism guarantee against current native facts.""" + context = plan.execution_context + communication = runtime_plan.communication + planned = runtime_plan.determinism.assumptions + provider_facts = { + "rank_count": native_facts.get("mpi_ranks"), + "device": native_facts.get("kokkos_device"), + "communicator": native_facts.get("communicator"), + "execution_backend": native_facts.get("kokkos_backend"), + "shared_space": native_facts.get("kokkos_shared_space"), + "stream_identity": native_facts.get("kokkos_stream"), + "reduction_order": [ + row.identity.token for row in communication.collectives + ], + "reduction_strategy": [ + "%s:%s" % (row.operation, row.strategy) + for row in communication.collectives + ], + } + actual = {} + for name in planned: + if name in provider_facts: + actual[name] = provider_facts[name] + continue + proof = context.backend.capabilities.get(name) + actual[name] = ( + None + if proof is None or not proof.known + else proof.require("runtime.%s" % name) + ) + runtime_plan.determinism.require_assumptions(actual) + + class _UniformNativeProvider(RuntimeExecutorProvider): def supports(self, install_plan: Any) -> bool: return _adaptive(install_plan) is False @@ -279,7 +317,9 @@ def install_runtime_executor(install_plan: Any, runtime_plan: Any = None) -> Any from pops.runtime._runtime_planning import require_runtime_plan_bundle runtime_plan = require_runtime_plan_bundle(plan, runtime_plan) - _require_supported_execution_context(plan) + native_facts = _native_runtime_facts() + _require_runtime_determinism(plan, runtime_plan, native_facts) + _require_supported_execution_context(plan, native_facts) matches = tuple(provider for provider in _PROVIDERS if provider.supports(plan)) if len(matches) != 1: raise ValueError( diff --git a/tests/python/unit/runtime/test_runtime_executor_context.py b/tests/python/unit/runtime/test_runtime_executor_context.py index 034ce4183..358a115a7 100644 --- a/tests/python/unit/runtime/test_runtime_executor_context.py +++ b/tests/python/unit/runtime/test_runtime_executor_context.py @@ -13,9 +13,15 @@ ExecutionResource, proven_serial_manifest, ) +from pops.identity import make_identity from pops.runtime import _multi_layout_executor as multi_executor from pops.runtime import _platform_manifest as platform_manifest from pops.runtime import _runtime_executor as executor +from pops.runtime import _runtime_planning as runtime_planning +from pops.runtime._runtime_plan_contracts import ( + DeterminismGuarantee, + RuntimePlanningError, +) from pops.runtime._runtime_planning import build_runtime_plans from tests.python.unit.runtime.test_runtime_planning import _install, _manifest @@ -89,6 +95,7 @@ def forbidden_constructor(*args, **kwargs): assert len(memory_spaces) == 1 facts = { "mpi_active": False, + "mpi_ranks": 1, "kokkos_backend": backend.capabilities["execution_backend"].require( "runtime.execution_backend" ), @@ -139,6 +146,62 @@ def forbidden_preflight(*args, **kwargs): assert calls == [] +def test_determinism_assumptions_are_rechecked_before_native_preflight(monkeypatch): + plan = SimpleNamespace(execution_context=SimpleNamespace()) + runtime_plan = SimpleNamespace( + determinism=DeterminismGuarantee( + "reproducible", + ("rank_count",), + {"rank_count": 1}, + {}, + make_identity("execution-context", {"test": "runtime-executor"}), + ), + communication=SimpleNamespace(collectives=()), + ) + calls = [] + + def forbidden_preflight(*args, **kwargs): + calls.append((args, kwargs)) + raise AssertionError("native preflight became reachable") + + monkeypatch.setattr(executor, "require_install_plan", lambda value: value) + monkeypatch.setattr(executor, "_require_supported_execution_context", forbidden_preflight) + monkeypatch.setattr( + runtime_planning, + "require_runtime_plan_bundle", + lambda _plan, value: value, + ) + monkeypatch.setattr( + executor, + "_native_runtime_facts", + lambda: { + "mpi_ranks": 2, + }, + ) + with pytest.raises(RuntimePlanningError) as error: + executor.install_runtime_executor(plan, runtime_plan) + assert error.value.code == "determinism_assumption_mismatch" + assert calls == [] + + +def test_matching_runtime_determinism_assumptions_are_consumed(): + guarantee = DeterminismGuarantee( + "reproducible", + ("rank_count",), + {"rank_count": 1}, + {}, + make_identity("execution-context", {"test": "matching-runtime-executor"}), + ) + executor._require_runtime_determinism( + SimpleNamespace(execution_context=SimpleNamespace()), + SimpleNamespace( + determinism=guarantee, + communication=SimpleNamespace(collectives=()), + ), + {"mpi_ranks": 1}, + ) + + def test_before_step_transfer_cycle_captures_every_native_source_before_any_apply(): From 3d65b151740b689e9a063a94a2b79d40fdb31844 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:55:56 +0200 Subject: [PATCH 044/109] feat(runtime): enforce single-layout plan projection --- .../runtime_instance_planning_contract.md | 3 + python/pops/runtime/_runtime_executor.py | 28 ++++++- .../runtime/test_runtime_executor_context.py | 79 +++++++++++++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) diff --git a/docs/design/runtime_instance_planning_contract.md b/docs/design/runtime_instance_planning_contract.md index 5b0d660a2..f21858060 100644 --- a/docs/design/runtime_instance_planning_contract.md +++ b/docs/design/runtime_instance_planning_contract.md @@ -83,6 +83,9 @@ bundle, mismatched authority or changed execution fact therefore fails before ex complete bundle is retained in the array-free `RuntimeInstance.inspect()` report under `instance.runtime_plan` so derived halos, transfers, collectives, fences, buffers and determinism assumptions remain reviewable rather than becoming hidden installation state. +Single-layout providers additionally require the exact ordered block/layout call projection, +layout-qualified halos, and the absence of unconsumed Transfer or mapping-provider routes before +constructing their sole native engine. For an accepted step, successful native finalization is an irreversible `native_finalized` boundary. The instance commits the engine state, accepted cursor set and consumer receipts across diff --git a/python/pops/runtime/_runtime_executor.py b/python/pops/runtime/_runtime_executor.py index 9db79f01e..8207649be 100644 --- a/python/pops/runtime/_runtime_executor.py +++ b/python/pops/runtime/_runtime_executor.py @@ -188,6 +188,31 @@ def _require_runtime_determinism( runtime_plan.determinism.require_assumptions(actual) +def _require_single_layout_runtime_plan(plan: Any, runtime_plan: Any) -> None: + """Require the exact call/layout projection consumed by one native engine.""" + layout_plan = plan.artifact.layout_plan + if len(layout_plan.layouts) != 1: + raise ValueError("single-layout native provider requires exactly one resolved layout") + layout_id = layout_plan.layouts[0].handle.qualified_id + assignments = { + row.subject.local_id: (row.subject_id, row.layout.qualified_id) + for row in layout_plan.assignments + if row.subject_kind == "block" + } + expected_calls = tuple(assignments[block.name] for block in plan.artifact.blocks) + actual_calls = tuple((row.block_id, row.layout_id) for row in runtime_plan.calls) + if actual_calls != expected_calls: + raise ValueError( + "RuntimePlanBundle calls differ from the single-layout InstallPlan projection" + ) + if runtime_plan.communication.transfers: + raise ValueError("single-layout native provider cannot consume layout Transfers") + if runtime_plan.resources.mapping_provider_ids: + raise ValueError("single-layout native provider cannot consume mapping providers") + if any(row.layout_id != layout_id for row in runtime_plan.communication.halos): + raise ValueError("RuntimePlanBundle halo differs from the installed single layout") + + class _UniformNativeProvider(RuntimeExecutorProvider): def supports(self, install_plan: Any) -> bool: return _adaptive(install_plan) is False @@ -201,6 +226,7 @@ def install(self, install_plan: Any, runtime_plan: Any = None) -> Any: return install_multi_layout_uniform(plan, runtime_plan) + _require_single_layout_runtime_plan(plan, runtime_plan) _require_native_geometry(plan) from pops.runtime._runtime_mesh_lowering import ( install_uniform_embedded_boundary, @@ -236,8 +262,8 @@ def supports(self, install_plan: Any) -> bool: return _adaptive(install_plan) is True def install(self, install_plan: Any, runtime_plan: Any = None) -> Any: - del runtime_plan plan = require_install_plan(install_plan) + _require_single_layout_runtime_plan(plan, runtime_plan) _require_native_geometry(plan) if plan.initial_condition_plan is None or plan.bootstrap_plan is None: raise ValueError( diff --git a/tests/python/unit/runtime/test_runtime_executor_context.py b/tests/python/unit/runtime/test_runtime_executor_context.py index 358a115a7..c31a1c7e7 100644 --- a/tests/python/unit/runtime/test_runtime_executor_context.py +++ b/tests/python/unit/runtime/test_runtime_executor_context.py @@ -202,6 +202,85 @@ def test_matching_runtime_determinism_assumptions_are_consumed(): ) +def _single_layout_projection(): + layout = SimpleNamespace(handle=SimpleNamespace(qualified_id="layout::primary")) + plan = SimpleNamespace( + artifact=SimpleNamespace( + blocks=(SimpleNamespace(name="fluid"),), + layout_plan=SimpleNamespace( + layouts=(layout,), + assignments=( + SimpleNamespace( + subject_kind="block", + subject_id="block::fluid", + subject=SimpleNamespace( + local_id="fluid", qualified_id="block::fluid" + ), + layout=layout.handle, + ), + ), + ), + ) + ) + runtime_plan = SimpleNamespace( + calls=(SimpleNamespace(block_id="block::fluid", layout_id="layout::primary"),), + communication=SimpleNamespace( + transfers=(), + halos=(SimpleNamespace(layout_id="layout::primary"),), + ), + resources=SimpleNamespace(mapping_provider_ids=()), + ) + return plan, runtime_plan + + +def test_single_layout_provider_consumes_exact_call_and_halo_projection(): + plan, runtime_plan = _single_layout_projection() + + executor._require_single_layout_runtime_plan(plan, runtime_plan) + + runtime_plan.calls[0].layout_id = "layout::other" + with pytest.raises(ValueError, match="calls differ"): + executor._require_single_layout_runtime_plan(plan, runtime_plan) + runtime_plan.calls[0].layout_id = "layout::primary" + + runtime_plan.communication.halos[0].layout_id = "layout::other" + with pytest.raises(ValueError, match="halo differs"): + executor._require_single_layout_runtime_plan(plan, runtime_plan) + + +@pytest.mark.parametrize("transfers,providers,match", [ + ((object(),), (), "layout Transfers"), + ((), ("pops://mapping/test",), "mapping providers"), +]) +def test_single_layout_provider_refuses_unconsumed_mapping_routes( + transfers, providers, match +): + plan, runtime_plan = _single_layout_projection() + runtime_plan.communication.transfers = transfers + runtime_plan.resources.mapping_provider_ids = providers + + with pytest.raises(ValueError, match=match): + executor._require_single_layout_runtime_plan(plan, runtime_plan) + + +@pytest.mark.parametrize( + "provider", + (executor._UniformNativeProvider(), executor._AdaptiveNativeProvider()), +) +def test_single_layout_providers_refuse_call_mismatch_before_geometry( + monkeypatch, provider +): + plan, runtime_plan = _single_layout_projection() + runtime_plan.calls[0].block_id = "block::other" + reached = [] + monkeypatch.setattr(executor, "require_install_plan", lambda value: value) + monkeypatch.setattr(executor, "_require_native_geometry", reached.append) + + with pytest.raises(ValueError, match="calls differ"): + provider.install(plan, runtime_plan) + assert reached == [] + + def test_before_step_transfer_cycle_captures_every_native_source_before_any_apply(): From 81973ae529e01e62950ea79e3ca72f4220209f09 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 04:58:53 +0200 Subject: [PATCH 045/109] feat(runtime): enforce multi-layout plan projection --- .../runtime_instance_planning_contract.md | 3 + python/pops/runtime/_multi_layout_executor.py | 28 ++++++++ .../runtime/test_runtime_executor_context.py | 65 +++++++++++++++++++ 3 files changed, 96 insertions(+) diff --git a/docs/design/runtime_instance_planning_contract.md b/docs/design/runtime_instance_planning_contract.md index f21858060..3cf2e9e97 100644 --- a/docs/design/runtime_instance_planning_contract.md +++ b/docs/design/runtime_instance_planning_contract.md @@ -86,6 +86,9 @@ assumptions remain reviewable rather than becoming hidden installation state. Single-layout providers additionally require the exact ordered block/layout call projection, layout-qualified halos, and the absence of unconsumed Transfer or mapping-provider routes before constructing their sole native engine. +The multi-layout Uniform provider likewise authenticates ordered block/layout calls and the exact +mapping-provider set backing its materialized Transfers before constructing child engines. It +refuses non-empty runtime halo plans until an explicit per-layout halo scheduler exists. For an accepted step, successful native finalization is an irreversible `native_finalized` boundary. The instance commits the engine state, accepted cursor set and consumer receipts across diff --git a/python/pops/runtime/_multi_layout_executor.py b/python/pops/runtime/_multi_layout_executor.py index bdba4c2a1..af3bb173f 100644 --- a/python/pops/runtime/_multi_layout_executor.py +++ b/python/pops/runtime/_multi_layout_executor.py @@ -153,6 +153,33 @@ def _require_conservative_cell_average_geometry(source: Any, target: Any) -> Non ) +def _require_runtime_plan_projection( + plan: Any, runtime_plan: Any, transfers: tuple[Any, ...] +) -> None: + """Require every multi-layout route the provider claims before engine construction.""" + layout_plan = plan.artifact.layout_plan + assignments = { + row.subject.local_id: (row.subject_id, row.layout.qualified_id) + for row in layout_plan.assignments + if row.subject_kind == "block" + } + expected_calls = tuple(assignments[block.name] for block in plan.artifact.blocks) + actual_calls = tuple((row.block_id, row.layout_id) for row in runtime_plan.calls) + if actual_calls != expected_calls: + raise ValueError( + "RuntimePlanBundle calls differ from the multi-layout InstallPlan projection" + ) + if runtime_plan.communication.halos: + raise NotImplementedError( + "multi-layout RuntimePlan halos require an explicit per-layout halo scheduler" + ) + expected_providers = tuple(sorted({row.provider_id for row in transfers})) + if runtime_plan.resources.mapping_provider_ids != expected_providers: + raise ValueError( + "RuntimePlanBundle mapping providers differ from the consumed Transfers" + ) + + def _require_runtime_plan_bundle(plan: Any, runtime_plan: Any) -> None: """Authenticate the exact bundle and its Transfer projection against one InstallPlan.""" from pops.runtime._runtime_plan_contracts import LayoutTransfer @@ -184,6 +211,7 @@ def _require_runtime_plan_bundle(plan: Any, runtime_plan: Any) -> None: raise ValueError( "RuntimePlanBundle Transfers differ from the authenticated compiled LayoutPlan" ) + _require_runtime_plan_projection(plan, runtime_plan, transfers) _require_unique_transfer_targets(transfers) diff --git a/tests/python/unit/runtime/test_runtime_executor_context.py b/tests/python/unit/runtime/test_runtime_executor_context.py index c31a1c7e7..f30ce9494 100644 --- a/tests/python/unit/runtime/test_runtime_executor_context.py +++ b/tests/python/unit/runtime/test_runtime_executor_context.py @@ -281,6 +281,71 @@ def test_single_layout_providers_refuse_call_mismatch_before_geometry( assert reached == [] +def _multi_layout_projection(): + primary = SimpleNamespace(qualified_id="layout::primary") + secondary = SimpleNamespace(qualified_id="layout::secondary") + blocks = ( + SimpleNamespace(name="fluid"), + SimpleNamespace(name="solid"), + ) + assignments = tuple( + SimpleNamespace( + subject_kind="block", + subject_id=block_id, + subject=SimpleNamespace(local_id=name), + layout=layout, + ) + for name, block_id, layout in ( + ("fluid", "block::fluid", primary), + ("solid", "block::solid", secondary), + ) + ) + plan = SimpleNamespace( + artifact=SimpleNamespace( + blocks=blocks, + layout_plan=SimpleNamespace(assignments=assignments), + ) + ) + transfer = SimpleNamespace(provider_id="pops://mapping/primary-secondary") + runtime_plan = SimpleNamespace( + calls=tuple( + SimpleNamespace(block_id=block_id, layout_id=layout.qualified_id) + for block_id, layout in ( + ("block::fluid", primary), + ("block::solid", secondary), + ) + ), + communication=SimpleNamespace(halos=()), + resources=SimpleNamespace( + mapping_provider_ids=("pops://mapping/primary-secondary",) + ), + ) + return plan, runtime_plan, (transfer,) + + +def test_multi_layout_provider_consumes_exact_call_and_mapping_projection(): + plan, runtime_plan, transfers = _multi_layout_projection() + + multi_executor._require_runtime_plan_projection(plan, runtime_plan, transfers) + + runtime_plan.calls[1].layout_id = "layout::primary" + with pytest.raises(ValueError, match="calls differ"): + multi_executor._require_runtime_plan_projection(plan, runtime_plan, transfers) + runtime_plan.calls[1].layout_id = "layout::secondary" + + runtime_plan.resources.mapping_provider_ids = ("pops://mapping/other",) + with pytest.raises(ValueError, match="mapping providers differ"): + multi_executor._require_runtime_plan_projection(plan, runtime_plan, transfers) + + +def test_multi_layout_provider_refuses_unconsumed_halo_plan(): + plan, runtime_plan, transfers = _multi_layout_projection() + runtime_plan.communication.halos = (object(),) + + with pytest.raises(NotImplementedError, match="explicit per-layout halo scheduler"): + multi_executor._require_runtime_plan_projection(plan, runtime_plan, transfers) + + def test_before_step_transfer_cycle_captures_every_native_source_before_any_apply(): From 3d3b250548b96b92845b035b0162a190f1a8c167 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:38:22 +0200 Subject: [PATCH 046/109] refactor(mpi): name interface field rank authority --- .../multiblock/interface_flux_scheduler.hpp | 13 ++++++++++--- ...multiblock_interface_communicator_fence.py | 19 +++++++++++++++++++ 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/include/pops/runtime/multiblock/interface_flux_scheduler.hpp b/include/pops/runtime/multiblock/interface_flux_scheduler.hpp index a240ab129..d82b2e7a7 100644 --- a/include/pops/runtime/multiblock/interface_flux_scheduler.hpp +++ b/include/pops/runtime/multiblock/interface_flux_scheduler.hpp @@ -113,9 +113,15 @@ class InterfaceFluxScheduler { MultiFab& right_state, const Geometry& right_geometry, const PopsExecutionContextV1& execution, InterfaceFluxEvaluatorFactory evaluator_factory) { - const bool collective_world = comm_active() && n_ranks() > 1; + // MultiFab/DistributionMapping still stores owners in the process-world rank space. Retain + // that storage authority under an explicit name for admission only: every numerical + // collective below runs on the communicator carried by ExecutionContext. Once field storage + // owns a communicator-relative rank space, this single compatibility seam can disappear. + const CommunicatorView field_rank_space = + comm_active() ? world_communicator_view() : CommunicatorView{}; + const bool collective_world = field_rank_space.active() && field_rank_space.size() > 1; const CommunicatorView admission_communicator = - collective_world ? world_communicator_view() : CommunicatorView{}; + collective_world ? field_rank_space : CommunicatorView{}; bool distributed = false; CommunicatorView execution_communicator; int communicator_rank = 0; @@ -157,7 +163,8 @@ class InterfaceFluxScheduler { "communicator/MPI_DOUBLE authority"); int communicator_relation = MPI_UNEQUAL; ::pops::detail::require_mpi_success( - MPI_Comm_compare(communicator, MPI_COMM_WORLD, &communicator_relation), + MPI_Comm_compare(communicator, field_rank_space.native_handle(), + &communicator_relation), "MPI_Comm_compare(interface field rank space)"); if (communicator_relation != MPI_IDENT && communicator_relation != MPI_CONGRUENT) throw std::invalid_argument( diff --git a/tests/python/architecture/test_multiblock_interface_communicator_fence.py b/tests/python/architecture/test_multiblock_interface_communicator_fence.py index d472965ef..6c2bc98b9 100644 --- a/tests/python/architecture/test_multiblock_interface_communicator_fence.py +++ b/tests/python/architecture/test_multiblock_interface_communicator_fence.py @@ -31,3 +31,22 @@ def test_interface_scheduler_hot_path_never_falls_back_to_mpi_world(): assert "MPI_COMM_WORLD" not in apply_one assert "const CommunicatorView& communicator" in consensus assert "prepared.communicator" in apply_one + + +def test_interface_scheduler_limits_world_rank_space_to_storage_admission(): + source = SCHEDULER.read_text(encoding="utf-8") + install = _function( + source, + "void install(AxisAlignedInterface route, MultiFab& left_state,", + ) + hot_path = source.split( + "void apply(const BoundaryEvaluationPoint& point,", + maxsplit=1, + )[1] + + assert "MPI_COMM_WORLD" not in source + assert source.count("world_communicator_view()") == 1 + assert "const CommunicatorView field_rank_space =" in install + assert "MPI_Comm_compare(communicator, field_rank_space.native_handle()" in install + assert "execution_communicator = CommunicatorView{communicator};" in install + assert "world_communicator_view()" not in hot_path From ab675cb122e2ced6d41b1186e5e177bbf92c4c72 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:38:31 +0200 Subject: [PATCH 047/109] docs(runtime): qualify communicator rank-space limit --- docs/design/native-capability-matrix.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/design/native-capability-matrix.md b/docs/design/native-capability-matrix.md index b13418093..d6a5a08c9 100644 --- a/docs/design/native-capability-matrix.md +++ b/docs/design/native-capability-matrix.md @@ -83,7 +83,11 @@ Supported native routes include: default-flux RHS evaluations must be simultaneous and contiguous in one Program point. `MPI_COMM_WORLD` layouts may distribute the two face decompositions independently: native C++ collectives reconstruct both traces, require a finite bit-identical shared flux on every rank, - then scatter only into locally owned residual cells. + then scatter only into locally owned residual cells. `MultiFab`/`DistributionMapping` ownership is + still indexed in the process-world rank space, so interface installation performs one explicit + admission comparison against that storage rank space. It then retains the world-congruent + communicator carried by `ExecutionContext`; trace, failure, flux and registry collectives never + reacquire the process world in the numerical hot path. Internal serial two-level work retains endpoint-qualified canonical fragments with exact Program weights and authoritative local substep duration. Those fragments authenticate the paired RHS update; they are not injected again into reflux because that would duplicate the same face flux. @@ -181,7 +185,11 @@ future validators: has ended; an embedding application retains its lifecycle. Python carries only the opaque native resource identity. - `parallel:custom_communicator`: caller-provided custom MPI communicators remain representable but - unavailable because the native engines expose no communicator-injection ABI. + unavailable at the public bind surface because field storage does not yet carry a + communicator-relative rank space. The native interface scheduler and layout-transfer consumers + can execute on an authenticated `MPI_IDENT`/`MPI_CONGRUENT` lane, but admission must still compare + that lane with the process-world-indexed field ownership. Subgroups and reordered communicators + are refused before kernel launch. - `precision:single_or_mixed`: `pops::Real` is `double`; single or mixed precision is unavailable. - `runtime:kokkos_lifecycle`: `runtime_environment_report()` exposes whether PoPS will lazily initialize Kokkos, has initialized it, or is attached to an externally initialized runtime. From 84a2f2be0bd5b74ac89c2e1c072d2bfb9b27e8e1 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 05:49:34 +0200 Subject: [PATCH 048/109] docs(runtime): map System communicator injection boundary --- docs/design/platform-manifest-contract.md | 42 +++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/docs/design/platform-manifest-contract.md b/docs/design/platform-manifest-contract.md index a2cba9910..adcedbbde 100644 --- a/docs/design/platform-manifest-contract.md +++ b/docs/design/platform-manifest-contract.md @@ -50,6 +50,48 @@ loaded, so plugins share the already-owned Kokkos/MPI runtimes. The external com records `MPI_COMM_WORLD` plus the MPI ABI proof and is checked against the explicit execution context at installation. +## Remaining native `System` communicator injection boundary + +The uniform native provider validates an `ExecutionContext` before launch, but it currently +constructs `System(SystemConfig)` before passing that authority into C++. `SystemDomain` therefore +builds its `DistributionMapping` from process-global rank queries, while `SystemFieldSolver`, +`ProgramContext`, `SystemProgramDriver`, field publication, and global field gathers still use +argument-free world collectives. This is an incomplete authority flow, not a missing collective +primitive: exact contract consensus already accepts a `CommunicatorView`, exact `SolveReport` +consensus accepts an `ExecutionLane`, and `SolveOutcome` already exposes `collective_lane`. + +Creating another private `MPI_COMM_WORLD` lane inside one of those consumers would not close the +contract. It would still capture process-global state, could order collectives differently from the +field owner, and would not prove that the lane rank space matches the process-world-indexed +`DistributionMapping`. + +The minimum native ABI cut is: + +1. Decode and validate the owned `PreparedExecutionContextV1` before constructing `System`; the + Python runtime provider must pass it to the native constructor/factory instead of attaching only + a Python `_execution_context` attribute after construction. +2. Store that authority for the complete `System::Impl` lifetime. Construct `SystemDomain` from its + explicit communicator rank/size, admit only `MPI_IDENT` or `MPI_CONGRUENT` with the current + process-world field rank space, and reject a subgroup or reordered communicator before allocating + fields. +3. Materialize one deterministically named, owning field-execution lane from that authenticated + communicator during construction. Pass it to `SystemFieldSolver`, its nested elliptic provider + registry, `ProgramContext`, `SystemProgramDriver`, field publication, and global gathers; none of + those consumers may create or rediscover a world lane in a solve/publication hot path. +4. Replace every argument-free reduction, rank query, ordered-byte consensus, and `SolveReport` + consensus in that graph with its lane-scoped overload. Return + `SolveOutcome::collective_lane` using the same lifetime-stable lane so accept/reject consensus and + publication hooks cannot escape onto a different communicator. +5. Keep the existing custom-communicator refusal until field storage owns a communicator-relative + rank space. A low-level test-only constructor may select an explicit serial/world authority, but + the final Python runtime path must not retain `System(SystemConfig)` as an implicit-world route. + +The closure proof must include an `MPI_Comm_dup` world-congruent launch, rank-local construction and +solve failures, divergent `SolveReport`/consumption actions, and refusal of wrong-rank-space +communicators before mutation. A source architecture fence must additionally show that the complete +uniform field-solve/publication graph contains no argument-free collective, `ExecutionLane::world`, +`world_communicator_view`, or raw `MPI_COMM_WORLD` capture. + `compile_native` has an explicit PE/COFF command and `_pops.lib` contract. By contrast, `compile_problem` and `compile_component` are currently fail-closed on Windows because their final authenticated PE/COFF symbol-inspection/publication pipeline does not yet exist. They never run a From 60ddc226cf43549f3990c634d62c272b94d09dab Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:00:20 +0200 Subject: [PATCH 049/109] release: prove the exact installed wheel --- scripts/final_release_contract.py | 1 + scripts/prove_installed_wheel.py | 195 ++++++++++++++++++ scripts/release_preflight.py | 71 ++++++- scripts/run_final_gate.py | 24 ++- .../architecture/test_final_release_gate.py | 142 +++++++++++++ 5 files changed, 431 insertions(+), 2 deletions(-) create mode 100644 scripts/prove_installed_wheel.py diff --git a/scripts/final_release_contract.py b/scripts/final_release_contract.py index 692536074..586954045 100644 --- a/scripts/final_release_contract.py +++ b/scripts/final_release_contract.py @@ -37,6 +37,7 @@ PYTHON_REQUIRED_SELECTION = "not mpi and not hdf5" REQUIRED_RELEASE_GATES = ( "official_build", + "installed_wheel", "doctor", "codesign", "native_conformance", diff --git a/scripts/prove_installed_wheel.py b/scripts/prove_installed_wheel.py new file mode 100644 index 000000000..d088df5d6 --- /dev/null +++ b/scripts/prove_installed_wheel.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Prove that the imported PoPS package is the exact retained release wheel.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence +import hashlib +import importlib.metadata +import json +from pathlib import Path +import sys +from typing import Any +from urllib.parse import unquote, urlparse +import zipfile + + +ROOT = Path(__file__).resolve().parents[1] +PROOF_SCHEMA_VERSION = 1 + + +class InstalledWheelProofError(RuntimeError): + """The retained wheel and the imported installation are not byte-identical.""" + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _sha256(path: Path) -> str: + return _sha256_bytes(path.read_bytes()) + + +def _outside_checkout(path: Path, *, label: str) -> Path: + resolved = path.resolve() + try: + resolved.relative_to(ROOT) + except ValueError: + return resolved + raise InstalledWheelProofError("%s must be outside the checkout: %s" % (label, resolved)) + + +def _direct_url_path(payload: Any) -> tuple[Path, str]: + if not isinstance(payload, dict) or set(payload) != {"archive_info", "url"}: + raise InstalledWheelProofError("installed distribution direct_url.json is malformed") + archive = payload["archive_info"] + if not isinstance(archive, dict): + raise InstalledWheelProofError("installed distribution archive_info is malformed") + hashes = archive.get("hashes") + if not isinstance(hashes, dict) or set(hashes) != {"sha256"}: + raise InstalledWheelProofError("installed distribution lacks one exact sha256 archive hash") + digest = hashes["sha256"] + if not isinstance(digest, str) or len(digest) != 64: + raise InstalledWheelProofError("installed distribution archive sha256 is malformed") + parsed = urlparse(payload["url"]) + if parsed.scheme != "file" or parsed.netloc not in {"", "localhost"}: + raise InstalledWheelProofError("installed distribution did not originate from a local wheel") + return Path(unquote(parsed.path)).resolve(), digest + + +def build_proof( + wheel: Path, + *, + package_file: Path, + native_extension: Path, + distribution_root: Path, + python_executable: Path, + installed_version: str, + direct_url: Any, +) -> dict[str, Any]: + """Authenticate one installed distribution against one exact wheel archive.""" + + retained = _outside_checkout(wheel, label="retained wheel") + package = _outside_checkout(package_file, label="installed package") + extension = _outside_checkout(native_extension, label="installed native extension") + distribution = _outside_checkout(distribution_root, label="installed distribution") + if retained.suffix != ".whl" or not retained.is_file(): + raise InstalledWheelProofError("retained wheel is not a readable .whl file") + for label, path in (("installed package", package), ("installed native extension", extension)): + if not path.is_file(): + raise InstalledWheelProofError("%s is not a readable file: %s" % (label, path)) + if not distribution.is_dir(): + raise InstalledWheelProofError( + "installed distribution root is not a directory: %s" % distribution + ) + if not isinstance(installed_version, str) or not installed_version: + raise InstalledWheelProofError("installed distribution version is empty") + + wheel_digest = _sha256(retained) + direct_path, direct_digest = _direct_url_path(direct_url) + if direct_path != retained or direct_digest != wheel_digest: + raise InstalledWheelProofError( + "installed distribution direct URL does not authenticate the retained wheel" + ) + + try: + with zipfile.ZipFile(retained) as archive: + names = archive.namelist() + native_members = [ + name + for name in names + if name.startswith("pops/") and Path(name).name.startswith("_pops.") + and name.endswith((".so", ".pyd")) + ] + metadata_members = [name for name in names if name.endswith(".dist-info/METADATA")] + if len(native_members) != 1: + raise InstalledWheelProofError( + "retained wheel must contain exactly one pops._pops extension" + ) + if len(metadata_members) != 1: + raise InstalledWheelProofError( + "retained wheel must contain exactly one METADATA record" + ) + native_member = native_members[0] + native_digest = _sha256_bytes(archive.read(native_member)) + metadata = archive.read(metadata_members[0]).decode("utf-8") + except (OSError, UnicodeDecodeError, zipfile.BadZipFile) as exc: + raise InstalledWheelProofError("retained wheel is unreadable: %s" % exc) from exc + + metadata_version = next( + ( + line.split(": ", 1)[1] + for line in metadata.splitlines() + if line.startswith("Version: ") + ), + None, + ) + if metadata_version != installed_version: + raise InstalledWheelProofError( + "installed distribution version disagrees with retained wheel metadata" + ) + installed_native_digest = _sha256(extension) + if installed_native_digest != native_digest: + raise InstalledWheelProofError( + "installed native extension is not byte-identical to the retained wheel member" + ) + + return { + "schema_version": PROOF_SCHEMA_VERSION, + "python_executable": str(python_executable.resolve()), + "distribution_root": str(distribution), + "package_file": str(package), + "native_extension": str(extension), + "native_member": native_member, + "native_sha256": native_digest, + "version": installed_version, + "wheel_path": str(retained), + "wheel_sha256": wheel_digest, + } + + +def installed_wheel_proof(wheel: Path) -> dict[str, Any]: + """Resolve the live imported distribution and authenticate it against ``wheel``.""" + + import pops + from pops import _pops + + distribution = importlib.metadata.distribution("pops") + direct_url_text = distribution.read_text("direct_url.json") + if direct_url_text is None: + raise InstalledWheelProofError( + "installed distribution has no direct_url.json for the retained wheel" + ) + try: + direct_url = json.loads(direct_url_text) + except json.JSONDecodeError as exc: + raise InstalledWheelProofError("installed direct_url.json is invalid JSON") from exc + if pops.__version__ != _pops.__version__ or pops.__version__ != distribution.version: + raise InstalledWheelProofError("installed Python/native/distribution versions disagree") + return build_proof( + wheel, + package_file=Path(pops.__file__), + native_extension=Path(_pops.__file__), + distribution_root=Path(distribution.locate_file("")), + python_executable=Path(sys.executable), + installed_version=distribution.version, + direct_url=direct_url, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel", required=True, type=Path) + args = parser.parse_args(argv) + try: + proof = installed_wheel_proof(args.wheel) + except (InstalledWheelProofError, OSError, ValueError) as exc: + print("installed wheel proof failed: %s" % exc, file=sys.stderr) + return 1 + print(json.dumps(proof, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index d9401c8fd..c71a99833 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -32,7 +32,7 @@ ROOT = Path(__file__).resolve().parents[1] GENERATED = ROOT / "python" / "pops" / "_generated_release_contract.py" REQUIRED_GATES = REQUIRED_RELEASE_GATES -EVIDENCE_SCHEMA_VERSION = 4 +EVIDENCE_SCHEMA_VERSION = 5 class PreflightError(RuntimeError): @@ -236,6 +236,74 @@ def _wheel_evidence(directory: Path, gates: dict[str, Any], contract: Any) -> No raise PreflightError("release wheel name/version disagrees with the release contract") +def _installed_wheel_evidence( + directory: Path, + gates: dict[str, Any], + contract: Any, + runtime: dict[str, str], +) -> None: + wheel = gates["official_build"]["evidence"]["wheel"] + retained = (directory / wheel["path"]).resolve() + row = gates["installed_wheel"] + evidence = row["evidence"] + expected = { + "schema_version", + "python_executable", + "distribution_root", + "package_file", + "native_extension", + "native_member", + "native_sha256", + "version", + "wheel_path", + "wheel_sha256", + } + if not isinstance(evidence, dict) or set(evidence) != expected: + raise PreflightError("installed wheel evidence is malformed") + if evidence["schema_version"] != 1: + raise PreflightError("installed wheel evidence schema is unsupported") + if evidence["version"] != contract.PACKAGE_VERSION: + raise PreflightError("installed wheel evidence version disagrees with release contract") + if Path(evidence["wheel_path"]).resolve() != retained \ + or evidence["wheel_sha256"] != wheel["sha256"]: + raise PreflightError("installed wheel evidence does not authenticate the retained wheel") + if evidence["python_executable"] != runtime["python_executable"] \ + or evidence["package_file"] != runtime["pops_file"] \ + or evidence["native_extension"] != runtime["native_extension"]: + raise PreflightError("installed wheel evidence belongs to another runtime") + + commands = row["commands"] + logs = _command_evidence(directory, commands, gate="installed_wheel") + if len(logs) != 2: + raise PreflightError("installed wheel gate requires reinstall and proof transcripts") + install_suffix = [ + "python", + "-m", + "pip", + "install", + "--force-reinstall", + "--no-deps", + str(retained), + ] + proof_suffix = [ + "python", + "scripts/prove_installed_wheel.py", + "--wheel", + str(retained), + ] + if commands[0]["argv"][-len(install_suffix):] != install_suffix \ + or commands[1]["argv"][-len(proof_suffix):] != proof_suffix: + raise PreflightError("installed wheel gate did not reinstall and prove the retained wheel") + try: + with zipfile.ZipFile(retained) as archive: + member = evidence["native_member"] + member_digest = hashlib.sha256(archive.read(member)).hexdigest() + except (KeyError, OSError, zipfile.BadZipFile) as exc: + raise PreflightError("installed wheel native member is unreadable: %s" % exc) from exc + if member_digest != evidence["native_sha256"]: + raise PreflightError("installed wheel native member hash drifted") + + def _examples_evidence(directory: Path, gates: dict[str, Any]) -> None: examples = gates["examples"]["evidence"] reopen = gates["artifact_reopen"]["evidence"] @@ -340,6 +408,7 @@ def _evidence(path: Path, contract: Any, commit: str, runtime: dict[str, str]) - else: _command_evidence(directory, commands, gate=name) _wheel_evidence(directory, gates, contract) + _installed_wheel_evidence(directory, gates, contract, runtime) for name in ("native_conformance", "python_conformance"): evidence = gates[name]["evidence"] expected = {"required_lane"} if name == "native_conformance" \ diff --git a/scripts/run_final_gate.py b/scripts/run_final_gate.py index efddb3463..96b0f4846 100644 --- a/scripts/run_final_gate.py +++ b/scripts/run_final_gate.py @@ -38,7 +38,7 @@ ROOT = Path(__file__).resolve().parents[1] -EVIDENCE_SCHEMA_VERSION = 4 +EVIDENCE_SCHEMA_VERSION = 5 REQUIRED_GATES = REQUIRED_RELEASE_GATES @@ -190,6 +190,16 @@ def _runtime_provenance() -> dict[str, str]: return payload +def _json_evidence(stdout: str, *, gate: str) -> dict[str, Any]: + try: + payload = json.loads(stdout) + except json.JSONDecodeError as exc: + raise FinalGateError("%s evidence was not JSON: %s" % (gate, stdout[-4000:])) from exc + if not isinstance(payload, dict): + raise FinalGateError("%s evidence must be a JSON object" % gate) + return payload + + def _contract() -> tuple[str, str]: generated = ROOT / "python" / "pops" / "_generated_release_contract.py" specification = importlib.util.spec_from_file_location("_final_release_contract", generated) @@ -465,6 +475,18 @@ def main(argv: Sequence[str] | None = None) -> int: "size": wheel.stat().st_size, }, } + recorder.run("installed_wheel", _conda_command([ + "python", "-m", "pip", "install", "--force-reinstall", "--no-deps", str(wheel), + ])) + installed_wheel_stdout = recorder.run( + "installed_wheel", + _conda_command([ + "python", "scripts/prove_installed_wheel.py", "--wheel", str(wheel), + ]), + ) + recorder.rows["installed_wheel"]["evidence"] = _json_evidence( + installed_wheel_stdout, gate="installed_wheel" + ) recorder.run("official_build", _conda_command(["cmake", "--preset", "serial"])) recorder.run("official_build", _conda_command(["cmake", "--build", "--preset", "serial"])) doctor_code = ( diff --git a/tests/python/architecture/test_final_release_gate.py b/tests/python/architecture/test_final_release_gate.py index afb28b249..02789d304 100644 --- a/tests/python/architecture/test_final_release_gate.py +++ b/tests/python/architecture/test_final_release_gate.py @@ -1,7 +1,9 @@ """Source-only contract checks for the final release gate (ADC-695).""" from __future__ import annotations +import hashlib import importlib.util +import json from pathlib import Path import sys import zipfile @@ -25,6 +27,7 @@ def _load(name: str, path: Path): contract = _load("final_release_contract", SCRIPTS / "final_release_contract.py") gate = _load("_final_release_gate_test", SCRIPTS / "run_final_gate.py") preflight = _load("_release_preflight_test", SCRIPTS / "release_preflight.py") +installed = _load("_installed_wheel_proof_test", SCRIPTS / "prove_installed_wheel.py") def _write_final_source_tree(root: Path) -> None: @@ -191,6 +194,145 @@ def test_release_evidence_authenticates_the_exact_retained_wheel(tmp_path): preflight._wheel_evidence(tmp_path, gates, release) +def test_installed_wheel_proof_requires_exact_native_member_and_direct_url(tmp_path): + wheel = tmp_path / "pops-0.3.0-cp312-cp312-macosx_11_0_arm64.whl" + native_bytes = b"exact wheel extension" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("pops/_pops.cpython-312-darwin.so", native_bytes) + archive.writestr( + "pops-0.3.0.dist-info/METADATA", + "Metadata-Version: 2.3\nName: PoPS\nVersion: 0.3.0\n", + ) + package = tmp_path / "site-packages" / "pops" / "__init__.py" + extension = package.parent / "_pops.cpython-312-darwin.so" + distribution = package.parents[1] + package.parent.mkdir(parents=True) + package.write_text("__version__ = '0.3.0'\n", encoding="utf-8") + extension.write_bytes(native_bytes) + wheel_sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() + direct_url = { + "archive_info": {"hashes": {"sha256": wheel_sha256}}, + "url": wheel.as_uri(), + } + + proof = installed.build_proof( + wheel, + package_file=package, + native_extension=extension, + distribution_root=distribution, + python_executable=Path(sys.executable), + installed_version="0.3.0", + direct_url=direct_url, + ) + + assert proof["wheel_sha256"] == wheel_sha256 + assert proof["native_sha256"] == hashlib.sha256(native_bytes).hexdigest() + extension.write_bytes(b"not the retained wheel") + with pytest.raises(installed.InstalledWheelProofError, match="not byte-identical"): + installed.build_proof( + wheel, + package_file=package, + native_extension=extension, + distribution_root=distribution, + python_executable=Path(sys.executable), + installed_version="0.3.0", + direct_url=direct_url, + ) + + +def test_release_preflight_authenticates_installed_wheel_proof_and_transcripts(tmp_path): + wheel = tmp_path / "wheels" / "pops-0.3.0-cp312-cp312-macosx_11_0_arm64.whl" + wheel.parent.mkdir() + native_member = "pops/_pops.cpython-312-darwin.so" + native_bytes = b"exact wheel extension" + with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr(native_member, native_bytes) + archive.writestr( + "pops-0.3.0.dist-info/METADATA", + "Metadata-Version: 2.3\nName: PoPS\nVersion: 0.3.0\n", + ) + runtime = { + "python_executable": "/proof/bin/python", + "pops_file": "/proof/site-packages/pops/__init__.py", + "native_extension": "/proof/site-packages/pops/_pops.so", + "native_sha256": "post-sign-runtime-digest", + } + wheel_sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() + commands = [] + command_argvs = ( + [ + "/proof/conda", + "run", + "python", + "-m", + "pip", + "install", + "--force-reinstall", + "--no-deps", + str(wheel), + ], + [ + "/proof/conda", + "run", + "python", + "scripts/prove_installed_wheel.py", + "--wheel", + str(wheel), + ], + ) + for index, argv in enumerate(command_argvs, 1): + log = tmp_path / "logs" / f"{index:02d}_installed_wheel.log" + log.parent.mkdir(exist_ok=True) + log.write_text(json.dumps({"ok": True}), encoding="utf-8") + commands.append( + { + "argv": argv, + "log": str(log.relative_to(tmp_path)), + "sha256": hashlib.sha256(log.read_bytes()).hexdigest(), + } + ) + gates = { + "official_build": { + "evidence": { + "wheel": { + "path": str(wheel.relative_to(tmp_path)), + "sha256": wheel_sha256, + "size": wheel.stat().st_size, + }, + }, + }, + "installed_wheel": { + "commands": commands, + "evidence": { + "schema_version": 1, + "python_executable": runtime["python_executable"], + "distribution_root": "/proof/site-packages", + "package_file": runtime["pops_file"], + "native_extension": runtime["native_extension"], + "native_member": native_member, + "native_sha256": hashlib.sha256(native_bytes).hexdigest(), + "version": "0.3.0", + "wheel_path": str(wheel), + "wheel_sha256": wheel_sha256, + }, + }, + } + release = type("ReleaseContract", (), {"PACKAGE_VERSION": "0.3.0"}) + + preflight._installed_wheel_evidence(tmp_path, gates, release, runtime) + gates["installed_wheel"]["evidence"]["native_sha256"] = "0" * 64 + with pytest.raises(preflight.PreflightError, match="native member hash drifted"): + preflight._installed_wheel_evidence(tmp_path, gates, release, runtime) + + +def test_installed_wheel_gate_precedes_codesign_and_conformance(): + gates = contract.REQUIRED_RELEASE_GATES + + assert gates.index("official_build") < gates.index("installed_wheel") + assert gates.index("installed_wheel") < gates.index("codesign") + assert gates.index("codesign") < gates.index("native_conformance") + + def test_tag_release_cannot_race_or_bypass_supported_matrix_wheel_and_final_gate(): release = (ROOT / ".github" / "workflows" / "release.yml").read_text() wheels = (ROOT / ".github" / "workflows" / "wheels.yml").read_text() From 83158f6bd240824b1b20e4836b4574edc24e579b Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:02:02 +0200 Subject: [PATCH 050/109] release: bind codesign evidence to runtime bytes --- scripts/codesign_pops_extensions.py | 33 ++++++++++++- scripts/final_release_contract.py | 2 +- scripts/release_preflight.py | 35 +++++++++++++- scripts/run_final_gate.py | 13 ++++-- .../test_codesign_build_scripts.py | 22 +++++++++ .../architecture/test_final_release_gate.py | 46 +++++++++++++++++++ 6 files changed, 144 insertions(+), 7 deletions(-) diff --git a/scripts/codesign_pops_extensions.py b/scripts/codesign_pops_extensions.py index 78113fe54..2a054e3d3 100755 --- a/scripts/codesign_pops_extensions.py +++ b/scripts/codesign_pops_extensions.py @@ -7,13 +7,19 @@ from __future__ import annotations import argparse +from collections.abc import Sequence +import hashlib import importlib.machinery import importlib.util +import json from pathlib import Path import shutil import subprocess import sys -from typing import Sequence +from typing import Any + + +CODESIGN_EVIDENCE_SCHEMA_VERSION = 1 class CodesignError(RuntimeError): @@ -97,17 +103,40 @@ def codesign_imported_extensions(*, if_present: bool = False) -> tuple[Path, ... return extensions +def codesign_evidence(extensions: Sequence[Path]) -> dict[str, Any]: + """Describe the exact post-sign extension bytes authenticated by this process.""" + return { + "schema_version": CODESIGN_EVIDENCE_SCHEMA_VERSION, + "platform": sys.platform, + "extensions": [ + { + "path": str(extension.resolve()), + "sha256": hashlib.sha256(extension.read_bytes()).hexdigest(), + "signature": "adhoc", + } + for extension in extensions + ], + } + + def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "--if-present", action="store_true", help="skip only when the pops package is absent (a present package without _pops fails)") + parser.add_argument( + "--json", action="store_true", + help="print machine-authenticated post-sign paths and hashes") args = parser.parse_args(argv) try: extensions = codesign_imported_extensions(if_present=args.if_present) - except CodesignError as error: + evidence = codesign_evidence(extensions) + except (CodesignError, OSError) as error: print("ERROR: %s" % error, file=sys.stderr) return 1 + if args.json: + print(json.dumps(evidence, sort_keys=True)) + return 0 if sys.platform == "darwin": if extensions: for extension in extensions: diff --git a/scripts/final_release_contract.py b/scripts/final_release_contract.py index 586954045..39658bcde 100644 --- a/scripts/final_release_contract.py +++ b/scripts/final_release_contract.py @@ -38,8 +38,8 @@ REQUIRED_RELEASE_GATES = ( "official_build", "installed_wheel", - "doctor", "codesign", + "doctor", "native_conformance", "python_conformance", "examples", diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index c71a99833..2f55668a3 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -32,7 +32,7 @@ ROOT = Path(__file__).resolve().parents[1] GENERATED = ROOT / "python" / "pops" / "_generated_release_contract.py" REQUIRED_GATES = REQUIRED_RELEASE_GATES -EVIDENCE_SCHEMA_VERSION = 5 +EVIDENCE_SCHEMA_VERSION = 6 class PreflightError(RuntimeError): @@ -304,6 +304,38 @@ def _installed_wheel_evidence( raise PreflightError("installed wheel native member hash drifted") +def _codesign_evidence( + directory: Path, + gates: dict[str, Any], + runtime: dict[str, str], +) -> None: + row = gates["codesign"] + evidence = row["evidence"] + if not isinstance(evidence, dict) or set(evidence) != { + "schema_version", "platform", "extensions"}: + raise PreflightError("codesign evidence is malformed") + if evidence["schema_version"] != 1 or evidence["platform"] != "darwin": + raise PreflightError("codesign evidence must authenticate the Darwin release lane") + extensions = evidence["extensions"] + if not isinstance(extensions, list) or len(extensions) != 1: + raise PreflightError("codesign evidence must authenticate exactly one extension") + extension = extensions[0] + if not isinstance(extension, dict) or set(extension) != { + "path", "sha256", "signature"}: + raise PreflightError("codesign extension evidence is malformed") + if extension != { + "path": runtime["native_extension"], + "sha256": runtime["native_sha256"], + "signature": "adhoc", + }: + raise PreflightError("codesign evidence does not authenticate the live native extension") + commands = row["commands"] + logs = _command_evidence(directory, commands, gate="codesign") + suffix = ["python", "scripts/codesign_pops_extensions.py", "--json"] + if len(logs) != 1 or commands[0]["argv"][-len(suffix):] != suffix: + raise PreflightError("codesign gate did not run the exact structured verifier") + + def _examples_evidence(directory: Path, gates: dict[str, Any]) -> None: examples = gates["examples"]["evidence"] reopen = gates["artifact_reopen"]["evidence"] @@ -409,6 +441,7 @@ def _evidence(path: Path, contract: Any, commit: str, runtime: dict[str, str]) - _command_evidence(directory, commands, gate=name) _wheel_evidence(directory, gates, contract) _installed_wheel_evidence(directory, gates, contract, runtime) + _codesign_evidence(directory, gates, runtime) for name in ("native_conformance", "python_conformance"): evidence = gates[name]["evidence"] expected = {"required_lane"} if name == "native_conformance" \ diff --git a/scripts/run_final_gate.py b/scripts/run_final_gate.py index 96b0f4846..9811cee4f 100644 --- a/scripts/run_final_gate.py +++ b/scripts/run_final_gate.py @@ -38,7 +38,7 @@ ROOT = Path(__file__).resolve().parents[1] -EVIDENCE_SCHEMA_VERSION = 5 +EVIDENCE_SCHEMA_VERSION = 6 REQUIRED_GATES = REQUIRED_RELEASE_GATES @@ -487,6 +487,15 @@ def main(argv: Sequence[str] | None = None) -> int: recorder.rows["installed_wheel"]["evidence"] = _json_evidence( installed_wheel_stdout, gate="installed_wheel" ) + codesign_stdout = recorder.run( + "codesign", + _conda_command([ + "python", "scripts/codesign_pops_extensions.py", "--json", + ]), + ) + recorder.rows["codesign"]["evidence"] = _json_evidence( + codesign_stdout, gate="codesign" + ) recorder.run("official_build", _conda_command(["cmake", "--preset", "serial"])) recorder.run("official_build", _conda_command(["cmake", "--build", "--preset", "serial"])) doctor_code = ( @@ -497,8 +506,6 @@ def main(argv: Sequence[str] | None = None) -> int: "print('doctor package=' + pops.__version__)" ) recorder.run("doctor", _conda_command(["python", "-c", doctor_code])) - recorder.run("codesign", _conda_command( - ["python", "scripts/codesign_pops_extensions.py"])) ctest_dir = _resolve_ctest_dir(args.ctest_dir) native_junit = evidence_root / "reports" / "native-conformance.xml" diff --git a/tests/python/architecture/test_codesign_build_scripts.py b/tests/python/architecture/test_codesign_build_scripts.py index 2045e8fbb..99807c62e 100644 --- a/tests/python/architecture/test_codesign_build_scripts.py +++ b/tests/python/architecture/test_codesign_build_scripts.py @@ -1,6 +1,7 @@ """ADC-647 source-only tests for post-install Darwin code-signing.""" from __future__ import annotations +import hashlib import importlib.util from pathlib import Path import subprocess @@ -79,6 +80,27 @@ def run(command, **kwargs): ] +def test_structured_evidence_binds_the_post_sign_extension_bytes(tmp_path, monkeypatch): + helper = _helper() + extension = tmp_path / "_pops.so" + extension.write_bytes(b"signed extension") + monkeypatch.setattr(helper.sys, "platform", "darwin") + + evidence = helper.codesign_evidence((extension,)) + + assert evidence == { + "schema_version": 1, + "platform": "darwin", + "extensions": [ + { + "path": str(extension.resolve()), + "sha256": hashlib.sha256(extension.read_bytes()).hexdigest(), + "signature": "adhoc", + } + ], + } + + @pytest.mark.parametrize("failure_call", [0, 1]) def test_darwin_codesign_or_verification_failure_is_explicit( tmp_path, monkeypatch, failure_call, diff --git a/tests/python/architecture/test_final_release_gate.py b/tests/python/architecture/test_final_release_gate.py index 02789d304..29d0f076b 100644 --- a/tests/python/architecture/test_final_release_gate.py +++ b/tests/python/architecture/test_final_release_gate.py @@ -330,9 +330,55 @@ def test_installed_wheel_gate_precedes_codesign_and_conformance(): assert gates.index("official_build") < gates.index("installed_wheel") assert gates.index("installed_wheel") < gates.index("codesign") + assert gates.index("codesign") < gates.index("doctor") assert gates.index("codesign") < gates.index("native_conformance") +def test_release_preflight_binds_codesign_to_live_runtime(tmp_path): + log = tmp_path / "logs" / "codesign.log" + log.parent.mkdir() + log.write_text('{"platform": "darwin"}\n', encoding="utf-8") + runtime = { + "python_executable": "/proof/bin/python", + "pops_file": "/proof/site-packages/pops/__init__.py", + "native_extension": "/proof/site-packages/pops/_pops.so", + "native_sha256": "a" * 64, + } + gates = { + "codesign": { + "commands": [ + { + "argv": [ + "/proof/conda", + "run", + "python", + "scripts/codesign_pops_extensions.py", + "--json", + ], + "log": str(log.relative_to(tmp_path)), + "sha256": hashlib.sha256(log.read_bytes()).hexdigest(), + } + ], + "evidence": { + "schema_version": 1, + "platform": "darwin", + "extensions": [ + { + "path": runtime["native_extension"], + "sha256": runtime["native_sha256"], + "signature": "adhoc", + } + ], + }, + }, + } + + preflight._codesign_evidence(tmp_path, gates, runtime) + gates["codesign"]["evidence"]["extensions"][0]["sha256"] = "b" * 64 + with pytest.raises(preflight.PreflightError, match="live native extension"): + preflight._codesign_evidence(tmp_path, gates, runtime) + + def test_tag_release_cannot_race_or_bypass_supported_matrix_wheel_and_final_gate(): release = (ROOT / ".github" / "workflows" / "release.yml").read_text() wheels = (ROOT / ".github" / "workflows" / "wheels.yml").read_text() From 0d8c0158fd7ed9bf1fc6aeff63025dfa873cc189 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:05:16 +0200 Subject: [PATCH 051/109] feat(runtime): retain automatic balance evidence per attempt --- .../runtime/program/program_runtime_state.hpp | 71 +++++++++++++++++++ src/runtime/amr/amr_system.cpp | 3 + src/runtime/system/system_impl.hpp | 3 + 3 files changed, 77 insertions(+) diff --git a/include/pops/runtime/program/program_runtime_state.hpp b/include/pops/runtime/program/program_runtime_state.hpp index a2ac587c9..e27984c07 100644 --- a/include/pops/runtime/program/program_runtime_state.hpp +++ b/include/pops/runtime/program/program_runtime_state.hpp @@ -143,6 +143,29 @@ struct HistoryManager { } }; +/// Attempt-local native balance evidence emitted by one exact runtime operator. +/// +/// The coordinate deliberately remains independent of a user-facing BalanceLedger route: native +/// operators know their qualified runtime block, hierarchy level and conservative component, while +/// the route-to-quantity selector is a separate planning authority. Keeping both identities +/// separate prevents a reflux correction from being silently relabelled as a complete balance. +struct AutomaticBalanceKey { + int runtime_block = -1; + int level = -1; + int component = -1; + std::string term; + + friend bool operator<(const AutomaticBalanceKey& left, const AutomaticBalanceKey& right) { + if (left.runtime_block != right.runtime_block) + return left.runtime_block < right.runtime_block; + if (left.level != right.level) + return left.level < right.level; + if (left.component != right.component) + return left.component < right.component; + return left.term < right.term; + } +}; + /// The compiled time-Program runtime state, extracted from the System / AmrSystem god-object (ADC-594). /// /// A plain aggregate: the owning Impl embeds ONE instance and routes every Program seam through it. The @@ -272,6 +295,11 @@ struct ProgramRuntimeState { /// consumers read it while the facade's outer transaction still retains U^n, so a missing term /// cannot silently reuse the preceding step. std::map step_balance_terms_; + /// Native operator contributions captured only for a due Balance attempt. These values are keyed + /// by their physical runtime coordinate instead of a user ledger route and are therefore not read + /// by accepted_balance_terms(). The owning facade snapshots this map with the rest of the attempt, + /// so rejection cannot leak automatic evidence into a retry. + std::map automatic_balance_terms_; /// Attempt-local outer accepted-step target used by ConsumerGraph-fused balance guards. Program /// substeps temporarily publish their window-start macro step through the facade, so generated /// balance code must not infer the public target from `macro_step()+1`. @@ -791,6 +819,14 @@ struct ProgramRuntimeState { throw std::invalid_argument(runtime + " requires one canonical five-term balance name"); } + static void require_automatic_balance_term(const std::string& term, const std::string& runtime) { + static constexpr std::array kTerms{"outward_boundary_flux", "sources", + "reflux", "projection"}; + if (std::find(kTerms.begin(), kTerms.end(), std::string_view(term)) == kTerms.end()) + throw std::invalid_argument(runtime + + " requires one native operator balance contribution name"); + } + /// Record a compiled-Program scalar. Ordinary P.record_scalar names remain inspectable after the /// step with last-write-wins semantics. The balance namespace has a separate typed sink. void record_diagnostic(const std::string& name, Real value) { @@ -816,6 +852,40 @@ struct ProgramRuntimeState { entry->second += value; } + /// Whether a compiled Program has actually emitted a due Balance route in this attempt. + /// + /// Generated balance records are cadence-guarded before their reductions. Reflux executes after + /// the Program body, so observing a non-empty authored mailbox here avoids every extra native + /// reduction on an off-cadence or replay step without introducing a second scheduler. + [[nodiscard]] bool automatic_balance_capture_due() const noexcept { + return !balance_replay_active_ && !step_balance_terms_.empty(); + } + + /// Accumulate one signed, metric-integrated native operator contribution. + /// + /// This is intentionally not accepted_balance_terms(): automatic evidence remains qualified by + /// block/level/component until a resolved quantity selector proves which BalanceLedger route owns + /// it. The separation is fail-closed and lets boundary/source/projection producers join the same + /// mailbox later without fabricating missing terms. + void record_automatic_balance_term(int runtime_block, int level, int component, + const std::string& term, Real value, + const std::string& runtime) { + if (!automatic_balance_capture_due()) + throw std::logic_error(runtime + + "::record_automatic_balance_term requires a due authored balance"); + if (runtime_block < 0 || level < 0 || component < 0) + throw std::invalid_argument( + runtime + "::record_automatic_balance_term requires non-negative coordinates"); + require_automatic_balance_term(term, runtime + "::record_automatic_balance_term"); + if (!std::isfinite(static_cast(value))) + throw std::invalid_argument(runtime + + "::record_automatic_balance_term requires a finite value"); + auto [entry, inserted] = automatic_balance_terms_.try_emplace( + AutomaticBalanceKey{runtime_block, level, component, term}, value); + if (!inserted) + entry->second += value; + } + /// Read the named diagnostic, FAIL-LOUD if the Program never recorded it. @p runtime names the /// Program subsystem setter in the message (not a generic getter). @throws std::out_of_range. Real diagnostic(const std::string& name, const std::string& runtime) const { @@ -833,6 +903,7 @@ struct ProgramRuntimeState { void begin_step_projection_report() { step_projections_.clear(); step_balance_terms_.clear(); + automatic_balance_terms_.clear(); balance_due_window_active_ = false; balance_due_target_step_ = 0; balance_step_completed_ = false; diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index d84124499..cf5d1d318 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -462,6 +462,7 @@ struct AmrSystem::Impl { int cadence_clock_restore_macro_step = 0; std::map program_diagnostics; std::map step_balance_terms; + std::map automatic_balance_terms; bool balance_step_completed = false; bool balance_program_was_due = false; pops::runtime::program::CacheManager cache; @@ -509,6 +510,7 @@ struct AmrSystem::Impl { cadence_clock_restore_macro_step = impl.program_.cadence_clock_restore_macro_step_; copy_value_map_into(program_diagnostics, impl.program_.diagnostics_); copy_value_map_into(step_balance_terms, impl.program_.step_balance_terms_); + copy_value_map_into(automatic_balance_terms, impl.program_.automatic_balance_terms_); balance_step_completed = impl.program_.balance_step_completed_; balance_program_was_due = impl.program_.balance_program_was_due_; // AMR currently owns its native cache/history rings inside AmrRuntime. These two shared @@ -542,6 +544,7 @@ struct AmrSystem::Impl { impl.program_.cadence_clock_restore_macro_step_ = cadence_clock_restore_macro_step; copy_value_map_into(impl.program_.diagnostics_, program_diagnostics); copy_value_map_into(impl.program_.step_balance_terms_, step_balance_terms); + copy_value_map_into(impl.program_.automatic_balance_terms_, automatic_balance_terms); impl.program_.balance_step_completed_ = balance_step_completed; impl.program_.balance_program_was_due_ = balance_program_was_due; impl.program_.cache_ = cache; diff --git a/src/runtime/system/system_impl.hpp b/src/runtime/system/system_impl.hpp index 13abe58bd..40fddf113 100644 --- a/src/runtime/system/system_impl.hpp +++ b/src/runtime/system/system_impl.hpp @@ -617,6 +617,7 @@ struct System::Impl { int cadence_clock_restore_macro_step; std::map program_diagnostics; std::map step_balance_terms; + std::map automatic_balance_terms; bool balance_step_completed; bool balance_program_was_due; pops::runtime::program::CacheManager cache; @@ -643,6 +644,7 @@ struct System::Impl { cadence_clock_restore_macro_step(impl.program_.cadence_clock_restore_macro_step_), program_diagnostics(impl.program_.diagnostics_), step_balance_terms(impl.program_.step_balance_terms_), + automatic_balance_terms(impl.program_.automatic_balance_terms_), balance_step_completed(impl.program_.balance_step_completed_), balance_program_was_due(impl.program_.balance_program_was_due_), cache(impl.program_.cache_), @@ -677,6 +679,7 @@ struct System::Impl { impl.program_.cadence_clock_restore_macro_step_ = cadence_clock_restore_macro_step; impl.program_.diagnostics_ = program_diagnostics; impl.program_.step_balance_terms_ = step_balance_terms; + impl.program_.automatic_balance_terms_ = automatic_balance_terms; impl.program_.balance_step_completed_ = balance_step_completed; impl.program_.balance_program_was_due_ = balance_program_was_due; impl.program_.cache_ = cache; From f2135ace5ee757b2ab61290761f65597a621d1dc Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:05:22 +0200 Subject: [PATCH 052/109] feat(amr): extract signed reflux balance corrections --- .../time/amr/levels/amr_patch_range.hpp | 17 +++++++++++++ .../time/amr/levels/amr_subcycling.hpp | 5 +++- .../pops/runtime/amr/amr_program_reflux.hpp | 11 +++++--- .../runtime/program/amr_program_context.hpp | 25 ++++++++++++++++--- 4 files changed, 51 insertions(+), 7 deletions(-) diff --git a/include/pops/numerics/time/amr/levels/amr_patch_range.hpp b/include/pops/numerics/time/amr/levels/amr_patch_range.hpp index 02a52a46b..82d8d1bf0 100644 --- a/include/pops/numerics/time/amr/levels/amr_patch_range.hpp +++ b/include/pops/numerics/time/amr/levels/amr_patch_range.hpp @@ -468,6 +468,23 @@ struct FluxRegister { device_fence(); all_reduce_sum_inplace(buf.data(), buf.size(), communicator); } + /// Sum the already-gathered sparse correction by conservative component. + /// + /// RefluxStorage is pinned host storage shared with device kernels. The fence makes the gathered + /// register host-readable; every communicator rank then traverses the same compact global order, + /// so this adds no second collective and produces the exact state increment applied below. + [[nodiscard]] std::vector component_sums(Real cell_measure) const { + if (!std::isfinite(static_cast(cell_measure)) || cell_measure <= Real(0)) + throw std::invalid_argument( + "FluxRegister component sum requires a finite positive cell measure"); + device_fence(); + std::vector result(static_cast(nc), Real(0)); + const std::size_t components = static_cast(nc); + for (std::size_t offset = 0; offset < buf.size(); offset += components) + for (std::size_t component = 0; component < components; ++component) + result[component] += cell_measure * buf[offset + component]; + return result; + } [[nodiscard]] std::size_t lookup_capacity() const noexcept { return cell_lookup.capacity(); } [[nodiscard]] std::size_t covered_cell_count() const noexcept { return cell_lookup.size(); } diff --git a/include/pops/numerics/time/amr/levels/amr_subcycling.hpp b/include/pops/numerics/time/amr/levels/amr_subcycling.hpp index 12292338a..0b4bde30e 100644 --- a/include/pops/numerics/time/amr/levels/amr_subcycling.hpp +++ b/include/pops/numerics/time/amr/levels/amr_subcycling.hpp @@ -824,7 +824,8 @@ class PreparedAmrProgramRefluxTransition { template void synchronize_integrated(MultiFab& parent_state, Real dx, Real dy, const CoarseStripRange& coarse_role, const FineStripRange& fine_role, - const CommunicatorView& communicator) { + const CommunicatorView& communicator, + std::vector* integrated_state_correction = nullptr) { validate_communicator_(communicator); using CoarseStrip = typename CoarseStripRange::value_type; using FineStrip = typename FineStripRange::value_type; @@ -882,6 +883,8 @@ class PreparedAmrProgramRefluxTransition { ncomp_); } correction_.gather(communicator); + if (integrated_state_correction != nullptr) + *integrated_state_correction = correction_.component_sums(dx * dy); for (int local_parent = 0; local_parent < parent_state.local_size(); ++local_parent) for_each_cell(parent_state.box(local_parent), detail::ApplyRefluxRegisterKernel{parent_state.fab(local_parent).array(), diff --git a/include/pops/runtime/amr/amr_program_reflux.hpp b/include/pops/runtime/amr/amr_program_reflux.hpp index 96ab1bd65..3e79b5af4 100644 --- a/include/pops/runtime/amr/amr_program_reflux.hpp +++ b/include/pops/runtime/amr/amr_program_reflux.hpp @@ -525,14 +525,19 @@ inline void sample_fine_role_strip(const MultiFab& state, const MultiFab& Fx, co /// per (cell,direction) (ADC-636 ownership: each C/F face is owned by the rank holding the covering fine /// patch), so the gather is associativity-free -> distributed == replicated bit-for-bit. inline void route_reflux_program(AmrRuntime& eng, std::size_t b, int k, const EdgeFlux& coarse_role, - const EdgeFlux& fine_role) { + const EdgeFlux& fine_role, + std::vector* integrated_state_correction = nullptr) { MultiFab& Uc = eng.level_state(b, k - 1); // the PARENT (coarse) live state we correct const BoxArray child_ba = eng.level_state(b, k).box_array(); // GLOBAL level-k patches - if (child_ba.size() == 0) + if (child_ba.size() == 0) { + if (integrated_state_correction != nullptr) + integrated_state_correction->assign(static_cast(Uc.ncomp()), Real(0)); return; + } const Geometry gc = eng.level_geom(k - 1); eng.prepared_reflux_transition(b, k).synchronize_integrated( - Uc, gc.dx(), gc.dy(), coarse_role.coarse, fine_role.fine, world_communicator_view()); + Uc, gc.dx(), gc.dy(), coarse_role.coarse, fine_role.fine, world_communicator_view(), + integrated_state_correction); } } // namespace detail diff --git a/include/pops/runtime/program/amr_program_context.hpp b/include/pops/runtime/program/amr_program_context.hpp index c22140dc3..05127b51c 100644 --- a/include/pops/runtime/program/amr_program_context.hpp +++ b/include/pops/runtime/program/amr_program_context.hpp @@ -1115,7 +1115,8 @@ class AmrProgramContext : public ProgramExecutionServices { amr::ClockStamp sync_clock = accepted; sync_clock.level = parent; for (int b = 0; b < n_blocks(); ++b) { - const std::size_t sb = static_cast(sys_block(b)); + const int runtime_block = sys_block(b); + const std::size_t sb = static_cast(runtime_block); if (capturing()) { sync_report_.push_back({parent, child, b, SyncPhase::Reflux, sync_clock}); const EdgeFlux coarse_role = reflux_flux_from_ledger_(b, parent, ledger_begin, ledger_end); @@ -1123,8 +1124,26 @@ class AmrProgramContext : public ProgramExecutionServices { if (coarse_role.empty() != fine_role.empty()) throw std::runtime_error( "AMR conservative ledger contains only one side of a parent/child flux pair"); - if (!coarse_role.empty()) - pops::detail::route_reflux_program(*eng_, sb, child, coarse_role, fine_role); + const bool capture_balance = + facade_->program_runtime_state_().automatic_balance_capture_due(); + std::vector integrated_reflux; + if (!coarse_role.empty()) { + pops::detail::route_reflux_program(*eng_, sb, child, coarse_role, fine_role, + capture_balance ? &integrated_reflux : nullptr); + } else if (capture_balance) { + integrated_reflux.assign(static_cast(eng_->level_state(sb, parent).ncomp()), + Real(0)); + } + if (capture_balance) { + const int components = eng_->level_state(sb, parent).ncomp(); + if (integrated_reflux.size() != static_cast(components)) + throw std::runtime_error( + "AMR automatic reflux balance contribution changed component width"); + for (int component = 0; component < components; ++component) + facade_->program_runtime_state_().record_automatic_balance_term( + runtime_block, parent, component, "reflux", + integrated_reflux[static_cast(component)], "AmrProgramContext"); + } } sync_report_.push_back({parent, child, b, SyncPhase::AverageDown, sync_clock}); eng_->average_down_level(sb, child); From 31d21432d373be50ff0416bf66332f7469bbc84b Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:05:26 +0200 Subject: [PATCH 053/109] test(architecture): fence automatic reflux balance evidence --- .../test_automatic_reflux_balance_fence.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 tests/python/architecture/test_automatic_reflux_balance_fence.py diff --git a/tests/python/architecture/test_automatic_reflux_balance_fence.py b/tests/python/architecture/test_automatic_reflux_balance_fence.py new file mode 100644 index 000000000..7b2c866a6 --- /dev/null +++ b/tests/python/architecture/test_automatic_reflux_balance_fence.py @@ -0,0 +1,99 @@ +"""ADC-686: automatic reflux evidence stays exact, sparse, and fail-closed.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +PROGRAM_STATE = ( + ROOT / "include" / "pops" / "runtime" / "program" / "program_runtime_state.hpp" +) +AMR_CONTEXT = ( + ROOT / "include" / "pops" / "runtime" / "program" / "amr_program_context.hpp" +) +AMR_REFLUX = ROOT / "include" / "pops" / "runtime" / "amr" / "amr_program_reflux.hpp" +AMR_SUBCYCLING = ( + ROOT / "include" / "pops" / "numerics" / "time" / "amr" / "levels" + / "amr_subcycling.hpp" +) +AMR_PATCH_RANGE = ( + ROOT / "include" / "pops" / "numerics" / "time" / "amr" / "levels" + / "amr_patch_range.hpp" +) +UNIFORM_IMPL = ROOT / "src" / "runtime" / "system" / "system_impl.hpp" +AMR_IMPL = ROOT / "src" / "runtime" / "amr" / "amr_system.cpp" + + +def _between(text: str, begin: str, end: str) -> str: + return text.split(begin, 1)[1].split(end, 1)[0] + + +def test_automatic_balance_mailbox_is_attempt_local_and_not_a_route_fallback() -> None: + state = PROGRAM_STATE.read_text() + assert "struct AutomaticBalanceKey" in state + assert "std::map automatic_balance_terms_;" in state + assert "automatic_balance_terms_.clear();" in state + assert "record_automatic_balance_term(" in state + assert "automatic_balance_capture_due()" in state + + accepted = _between( + state, + "std::map accepted_balance_terms(", + "void begin_balance_due_window(", + ) + assert "step_balance_terms_" in accepted + assert "automatic_balance_terms_" not in accepted + + uniform = UNIFORM_IMPL.read_text() + adaptive = AMR_IMPL.read_text() + for source in (uniform, adaptive): + assert "automatic_balance_terms" in source + assert "impl.program_.automatic_balance_terms_" in source + + +def test_reflux_integral_comes_from_the_gathered_sparse_correction() -> None: + register = AMR_PATCH_RANGE.read_text() + component_sums = _between( + register, + "[[nodiscard]] std::vector component_sums(", + "[[nodiscard]] std::size_t lookup_capacity()", + ) + assert "device_fence();" in component_sums + assert "cell_measure * buf[offset + component]" in component_sums + assert "all_reduce" not in component_sums + + transition = AMR_SUBCYCLING.read_text() + synchronize = _between( + transition, + "void synchronize_integrated(", + "\n private:", + ) + assert synchronize.index("correction_.gather(communicator);") < synchronize.index( + "correction_.component_sums(dx * dy)" + ) + assert synchronize.index("correction_.component_sums(dx * dy)") < synchronize.index( + "ApplyRefluxRegisterKernel" + ) + + route = AMR_REFLUX.read_text() + routing = _between(route, "inline void route_reflux_program(", "\n}\n\n} // namespace detail") + assert "std::vector* integrated_state_correction = nullptr" in routing + assert "integrated_state_correction);" in routing + + +def test_amr_records_reflux_before_average_down_only_when_balance_is_due() -> None: + context = AMR_CONTEXT.read_text() + synchronize = _between( + context, + "void synchronize_level_pair_(", + "void finalize_history_rotation_()", + ) + assert "automatic_balance_capture_due()" in synchronize + assert "record_automatic_balance_term(" in synchronize + assert '"reflux"' in synchronize + assert "reduce_sum(" not in synchronize + assert synchronize.index("route_reflux_program(") < synchronize.index( + "record_automatic_balance_term(" + ) + assert synchronize.index("record_automatic_balance_term(") < synchronize.index( + "SyncPhase::AverageDown" + ) From d30fd4351840abd6e9d47fee28f32cec568c650e Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:05:32 +0200 Subject: [PATCH 054/109] release: bind final examples to signed runtime --- scripts/release_preflight.py | 42 +++-- scripts/run_final_gate.py | 44 ++++- scripts/run_installed_example.py | 81 +++++++++ .../architecture/test_final_release_gate.py | 160 ++++++++++++++++++ 4 files changed, 312 insertions(+), 15 deletions(-) create mode 100644 scripts/run_installed_example.py diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index 2f55668a3..5ff58b793 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -32,7 +32,7 @@ ROOT = Path(__file__).resolve().parents[1] GENERATED = ROOT / "python" / "pops" / "_generated_release_contract.py" REQUIRED_GATES = REQUIRED_RELEASE_GATES -EVIDENCE_SCHEMA_VERSION = 6 +EVIDENCE_SCHEMA_VERSION = 7 class PreflightError(RuntimeError): @@ -336,7 +336,11 @@ def _codesign_evidence( raise PreflightError("codesign gate did not run the exact structured verifier") -def _examples_evidence(directory: Path, gates: dict[str, Any]) -> None: +def _examples_evidence( + directory: Path, + gates: dict[str, Any], + runtime: dict[str, str], +) -> None: examples = gates["examples"]["evidence"] reopen = gates["artifact_reopen"]["evidence"] restart = gates["strict_restart"]["evidence"] @@ -350,25 +354,41 @@ def _examples_evidence(directory: Path, gates: dict[str, Any]) -> None: logs = _command_evidence(directory, command_rows, gate="examples") if len(logs) != len(FINAL_EXAMPLES): raise PreflightError("final examples must have one execution transcript each") - for example in FINAL_EXAMPLES: + for index, example in enumerate(FINAL_EXAMPLES): key = example.as_posix() row = examples["examples"][key] - if not isinstance(row, dict) or set(row) != {"source_sha256", "stdout_sha256", "output_root"}: + if not isinstance(row, dict) or set(row) != { + "source_sha256", "stdout_sha256", "output_root", "runtime_sha256"}: raise PreflightError("release evidence %s is malformed" % key) if row["source_sha256"] != hashlib.sha256((ROOT / example).read_bytes()).hexdigest(): raise PreflightError("release evidence source drifted for %s" % key) if not isinstance(row["output_root"], str): raise PreflightError("release evidence output root is invalid for %s" % key) - matching = [log for log, command in zip(logs, command_rows, strict=True) - if key in " ".join(command["argv"])] - if len(matching) != 1: - raise PreflightError("release evidence has no unique command transcript for %s" % key) - transcript = matching[0].read_text(encoding="utf-8") + if row["runtime_sha256"] != runtime["native_sha256"]: + raise PreflightError("release evidence runtime digest drifted for %s" % key) + output_root = (directory / row["output_root"]).resolve() + expected_suffix = [ + "python", + "scripts/run_installed_example.py", + "--runtime-sha256", + runtime["native_sha256"], + "--example", + key, + "--", + "--output-dir", + str(output_root), + ] + command = command_rows[index]["argv"] + if command[-len(expected_suffix):] != expected_suffix: + raise PreflightError("release evidence command drifted for %s" % key) + transcript = logs[index].read_text(encoding="utf-8") if row["stdout_sha256"] != hashlib.sha256(transcript.encode("utf-8")).hexdigest(): raise PreflightError("release evidence stdout hash drifted for %s" % key) if any(marker not in transcript for marker in REQUIRED_PROOF_MARKERS): raise PreflightError("release evidence lacks restart/reopen proof output for %s" % key) - output_root = (directory / row["output_root"]).resolve() + runtime_marker = "PoPS release runtime | native_sha256=" + runtime["native_sha256"] + if transcript.count(runtime_marker) != 1: + raise PreflightError("release evidence runtime binding drifted for %s" % key) if not _inside(directory, output_root) or not output_root.is_dir(): raise PreflightError("release evidence output root is absent for %s" % key) reopened = reopen["examples"][key] @@ -462,7 +482,7 @@ def _evidence(path: Path, contract: Any, commit: str, runtime: dict[str, str]) - label="%s JUnit" % name) if gates["python_conformance"]["evidence"]["selection"] != PYTHON_REQUIRED_SELECTION: raise PreflightError("release evidence Python required-lane selection drifted") - _examples_evidence(directory, gates) + _examples_evidence(directory, gates, runtime) def main() -> int: diff --git a/scripts/run_final_gate.py b/scripts/run_final_gate.py index 9811cee4f..0fbbbf2bc 100644 --- a/scripts/run_final_gate.py +++ b/scripts/run_final_gate.py @@ -38,7 +38,7 @@ ROOT = Path(__file__).resolve().parents[1] -EVIDENCE_SCHEMA_VERSION = 6 +EVIDENCE_SCHEMA_VERSION = 7 REQUIRED_GATES = REQUIRED_RELEASE_GATES @@ -200,6 +200,24 @@ def _json_evidence(stdout: str, *, gate: str) -> dict[str, Any]: return payload +def _signed_runtime_sha256(evidence: dict[str, Any]) -> str: + if set(evidence) != {"schema_version", "platform", "extensions"} \ + or evidence["schema_version"] != 1 or evidence["platform"] != "darwin": + raise FinalGateError("codesign evidence is not the Darwin release proof") + extensions = evidence["extensions"] + if not isinstance(extensions, list) or len(extensions) != 1: + raise FinalGateError("codesign evidence must authenticate exactly one extension") + extension = extensions[0] + if not isinstance(extension, dict) or set(extension) != { + "path", "sha256", "signature"} or extension["signature"] != "adhoc": + raise FinalGateError("codesign extension evidence is malformed") + digest = extension["sha256"] + if not isinstance(digest, str) or len(digest) != 64 \ + or any(character not in "0123456789abcdef" for character in digest): + raise FinalGateError("codesign extension sha256 is malformed") + return digest + + def _contract() -> tuple[str, str]: generated = ROOT / "python" / "pops" / "_generated_release_contract.py" specification = importlib.util.spec_from_file_location("_final_release_contract", generated) @@ -373,7 +391,11 @@ def _reopen_npz_with_installed_runtime(recorder: Recorder, paths: Sequence[Path] ["python", "-c", code, *(str(path) for path in paths)])) -def _run_examples(recorder: Recorder) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: +def _run_examples( + recorder: Recorder, + *, + runtime_sha256: str, +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: results: dict[str, Any] = {} reopened: dict[str, Any] = {} restarted: dict[str, Any] = {} @@ -382,7 +404,17 @@ def _run_examples(recorder: Recorder) -> tuple[dict[str, Any], dict[str, Any], d destination = examples_root / example.stem stdout = recorder.run( "examples", - _conda_command(["python", str(example), "--output-dir", str(destination)]), + _conda_command([ + "python", + "scripts/run_installed_example.py", + "--runtime-sha256", + runtime_sha256, + "--example", + str(example), + "--", + "--output-dir", + str(destination), + ]), ) missing = [marker for marker in REQUIRED_PROOF_MARKERS if marker not in stdout] if missing: @@ -401,6 +433,7 @@ def _run_examples(recorder: Recorder) -> tuple[dict[str, Any], dict[str, Any], d "source_sha256": _sha256(ROOT / example), "stdout_sha256": hashlib.sha256(stdout.encode("utf-8")).hexdigest(), "output_root": str(destination.relative_to(recorder.root)), + "runtime_sha256": runtime_sha256, } return results, reopened, restarted @@ -529,7 +562,10 @@ def main(argv: Sequence[str] | None = None) -> int: "required_lane": _junit_summary(python_junit), "selection": PYTHON_REQUIRED_SELECTION, } - examples, reopened, restarted = _run_examples(recorder) + signed_runtime_sha256 = _signed_runtime_sha256( + recorder.rows["codesign"]["evidence"]) + examples, reopened, restarted = _run_examples( + recorder, runtime_sha256=signed_runtime_sha256) recorder.rows["examples"]["evidence"] = {"examples": examples} recorder.rows["artifact_reopen"]["evidence"] = {"examples": reopened} recorder.derived("strict_restart", {"examples": restarted}) diff --git a/scripts/run_installed_example.py b/scripts/run_installed_example.py new file mode 100644 index 000000000..67191b6a7 --- /dev/null +++ b/scripts/run_installed_example.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Run one release example only after authenticating its installed PoPS runtime.""" + +from __future__ import annotations + +import argparse +from collections.abc import Sequence +import hashlib +from pathlib import Path +import runpy +import sys + + +ROOT = Path(__file__).resolve().parents[1] +RUNTIME_MARKER = "PoPS release runtime | native_sha256=" + + +class InstalledExampleError(RuntimeError): + """The example is not bound to the expected installed native runtime.""" + + +def _outside_checkout(path: Path, *, label: str) -> Path: + resolved = path.resolve() + try: + resolved.relative_to(ROOT) + except ValueError: + return resolved + raise InstalledExampleError("%s must be outside the checkout: %s" % (label, resolved)) + + +def verify_installed_runtime(expected_sha256: str) -> str: + """Import PoPS once and return the authenticated native extension digest.""" + if len(expected_sha256) != 64 or any( + character not in "0123456789abcdef" for character in expected_sha256): + raise InstalledExampleError("expected native sha256 is malformed") + + import pops + from pops import _pops + + _outside_checkout(Path(pops.__file__), label="installed PoPS package") + extension = _outside_checkout( + Path(_pops.__file__), label="installed PoPS native extension") + digest = hashlib.sha256(extension.read_bytes()).hexdigest() + if digest != expected_sha256: + raise InstalledExampleError( + "installed native extension does not match signed release runtime") + if pops.__version__ != _pops.__version__: + raise InstalledExampleError("installed Python and native versions disagree") + return digest + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--runtime-sha256", required=True) + parser.add_argument("--example", required=True, type=Path) + parser.add_argument("example_args", nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + try: + digest = verify_installed_runtime(args.runtime_sha256) + example = args.example.resolve() + if not example.is_file(): + raise InstalledExampleError("release example is not a readable file: %s" % example) + try: + example.relative_to(ROOT) + except ValueError as exc: + raise InstalledExampleError( + "release example must belong to this checkout: %s" % example) from exc + forwarded = list(args.example_args) + if forwarded[:1] == ["--"]: + forwarded.pop(0) + print(RUNTIME_MARKER + digest, flush=True) + sys.argv = [str(example), *forwarded] + runpy.run_path(str(example), run_name="__main__") + return 0 + except (InstalledExampleError, OSError, ValueError) as exc: + print("installed example failed: %s" % exc, file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/python/architecture/test_final_release_gate.py b/tests/python/architecture/test_final_release_gate.py index 29d0f076b..e23a5dc6d 100644 --- a/tests/python/architecture/test_final_release_gate.py +++ b/tests/python/architecture/test_final_release_gate.py @@ -6,6 +6,7 @@ import json from pathlib import Path import sys +import types import zipfile import pytest @@ -28,6 +29,7 @@ def _load(name: str, path: Path): gate = _load("_final_release_gate_test", SCRIPTS / "run_final_gate.py") preflight = _load("_release_preflight_test", SCRIPTS / "release_preflight.py") installed = _load("_installed_wheel_proof_test", SCRIPTS / "prove_installed_wheel.py") +example_runner = _load("_installed_example_test", SCRIPTS / "run_installed_example.py") def _write_final_source_tree(root: Path) -> None: @@ -379,6 +381,164 @@ def test_release_preflight_binds_codesign_to_live_runtime(tmp_path): preflight._codesign_evidence(tmp_path, gates, runtime) +def test_installed_example_authenticates_native_bytes_before_execution( + monkeypatch, tmp_path, capsys, +): + package = tmp_path / "site-packages" / "pops" / "__init__.py" + extension = package.parent / "_pops.so" + package.parent.mkdir(parents=True) + package.write_text("", encoding="utf-8") + extension.write_bytes(b"signed release runtime") + native = types.ModuleType("pops._pops") + native.__file__ = str(extension) + native.__version__ = "1.0.0" + pops = types.ModuleType("pops") + pops.__file__ = str(package) + pops.__version__ = "1.0.0" + pops._pops = native + monkeypatch.setitem(sys.modules, "pops", pops) + monkeypatch.setitem(sys.modules, "pops._pops", native) + digest = hashlib.sha256(extension.read_bytes()).hexdigest() + + assert example_runner.verify_installed_runtime(digest) == digest + with pytest.raises(example_runner.InstalledExampleError, match="does not match"): + example_runner.verify_installed_runtime("0" * 64) + + monkeypatch.setattr(example_runner, "ROOT", tmp_path) + example = tmp_path / "example.py" + example.write_text( + "import sys\nprint('example_args=' + '|'.join(sys.argv[1:]))\n", + encoding="utf-8", + ) + monkeypatch.setattr(example_runner, "verify_installed_runtime", lambda expected: expected) + assert example_runner.main([ + "--runtime-sha256", + digest, + "--example", + str(example), + "--", + "--output-dir", + "/proof/output", + ]) == 0 + output = capsys.readouterr().out + assert example_runner.RUNTIME_MARKER + digest in output + assert "example_args=--output-dir|/proof/output" in output + + +def test_final_gate_rejects_incomplete_or_non_darwin_codesign_runtime(): + evidence = { + "schema_version": 1, + "platform": "darwin", + "extensions": [ + { + "path": "/proof/pops/_pops.so", + "sha256": "a" * 64, + "signature": "adhoc", + } + ], + } + + assert gate._signed_runtime_sha256(evidence) == "a" * 64 + evidence["platform"] = "linux" + with pytest.raises(gate.FinalGateError, match="Darwin release proof"): + gate._signed_runtime_sha256(evidence) + + +def test_release_preflight_requires_exact_runtime_bound_example_commands(tmp_path): + runtime = { + "python_executable": "/proof/bin/python", + "pops_file": "/proof/site-packages/pops/__init__.py", + "native_extension": "/proof/site-packages/pops/_pops.so", + "native_sha256": "c" * 64, + } + examples = {} + reopened = {} + restarted = {} + commands = [] + for index, example in enumerate(contract.FINAL_EXAMPLES, 1): + key = example.as_posix() + output_root = tmp_path / "examples" / example.stem + output_root.mkdir(parents=True) + hdf5 = output_root / "state.h5" + hdf5.write_bytes(b"\x89HDF\r\n\x1a\npayload") + npz = output_root / "state.npz" + with zipfile.ZipFile(npz, "w") as archive: + archive.writestr("state.npy", b"payload") + paraview = output_root / "state.vtu" + paraview.write_text("", encoding="utf-8") + checkpoint = output_root / "checkpoint.bin" + checkpoint.write_bytes(b"restart") + transcript = "\n".join( + [ + example_runner.RUNTIME_MARKER + runtime["native_sha256"], + *contract.REQUIRED_PROOF_MARKERS, + ] + ) + "\n" + log = tmp_path / "logs" / f"{index:02d}_examples.log" + log.parent.mkdir(exist_ok=True) + log.write_text(transcript, encoding="utf-8") + commands.append( + { + "argv": [ + "/proof/conda", + "run", + "python", + "scripts/run_installed_example.py", + "--runtime-sha256", + runtime["native_sha256"], + "--example", + key, + "--", + "--output-dir", + str(output_root), + ], + "log": str(log.relative_to(tmp_path)), + "sha256": hashlib.sha256(log.read_bytes()).hexdigest(), + } + ) + examples[key] = { + "source_sha256": hashlib.sha256((ROOT / example).read_bytes()).hexdigest(), + "stdout_sha256": hashlib.sha256(transcript.encode("utf-8")).hexdigest(), + "output_root": str(output_root.relative_to(tmp_path)), + "runtime_sha256": runtime["native_sha256"], + } + reopened[key] = { + "hdf5": [ + { + "path": hdf5.name, + "sha256": hashlib.sha256(hdf5.read_bytes()).hexdigest(), + } + ], + "npz": [ + { + "path": npz.name, + "sha256": hashlib.sha256(npz.read_bytes()).hexdigest(), + } + ], + "paraview": [ + { + "path": paraview.name, + "sha256": hashlib.sha256(paraview.read_bytes()).hexdigest(), + } + ], + } + restarted[key] = { + "checkpoint": str(checkpoint), + "tree_sha256": hashlib.sha256(checkpoint.read_bytes()).hexdigest(), + "proof_markers": list(contract.REQUIRED_PROOF_MARKERS), + } + gates = { + "examples": {"commands": commands, "evidence": {"examples": examples}}, + "artifact_reopen": {"evidence": {"examples": reopened}}, + "strict_restart": {"evidence": {"examples": restarted}}, + } + + preflight._examples_evidence(tmp_path, gates, runtime) + commands[0]["argv"][commands[0]["argv"].index(runtime["native_sha256"])] = "d" * 64 + with pytest.raises(preflight.PreflightError, match="command drifted"): + preflight._examples_evidence(tmp_path, gates, runtime) + + def test_tag_release_cannot_race_or_bypass_supported_matrix_wheel_and_final_gate(): release = (ROOT / ".github" / "workflows" / "release.yml").read_text() wheels = (ROOT / ".github" / "workflows" / "wheels.yml").read_text() From f9348aa00e44bb644c9ff95163b3c7adfff28d4d Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:12:05 +0200 Subject: [PATCH 055/109] release: authenticate wheel before native import --- scripts/prove_installed_wheel.py | 89 +++++++++++++++++-- scripts/release_preflight.py | 22 ++++- .../architecture/test_final_release_gate.py | 36 +++++++- 3 files changed, 136 insertions(+), 11 deletions(-) diff --git a/scripts/prove_installed_wheel.py b/scripts/prove_installed_wheel.py index d088df5d6..7d5d2f7fc 100644 --- a/scripts/prove_installed_wheel.py +++ b/scripts/prove_installed_wheel.py @@ -6,6 +6,7 @@ import argparse from collections.abc import Sequence import hashlib +import importlib.machinery import importlib.metadata import json from pathlib import Path @@ -16,7 +17,7 @@ ROOT = Path(__file__).resolve().parents[1] -PROOF_SCHEMA_VERSION = 1 +PROOF_SCHEMA_VERSION = 2 class InstalledWheelProofError(RuntimeError): @@ -58,6 +59,71 @@ def _direct_url_path(payload: Any) -> tuple[Path, str]: return Path(unquote(parsed.path)).resolve(), digest +def _wheel_payload_proof( + archive: zipfile.ZipFile, + *, + distribution_root: Path, +) -> tuple[int, str]: + """Authenticate every directly installed wheel member except mutable ``RECORD``.""" + + rows: list[str] = [] + for name in sorted(archive.namelist()): + if name.endswith("/") or name.endswith(".dist-info/RECORD"): + continue + if ".data/" in name: + raise InstalledWheelProofError( + "retained wheel uses an unsupported .data installation scheme" + ) + relative = Path(name) + installed = (distribution_root / relative).resolve() + try: + installed.relative_to(distribution_root) + except ValueError as exc: + raise InstalledWheelProofError( + "wheel member escapes the installed distribution root: %s" % name + ) from exc + if not installed.is_file(): + raise InstalledWheelProofError( + "installed distribution is missing wheel member %s" % name + ) + wheel_digest = _sha256_bytes(archive.read(name)) + if _sha256(installed) != wheel_digest: + raise InstalledWheelProofError( + "installed wheel member is not byte-identical: %s" % name + ) + rows.append("%s\0%s\n" % (name, wheel_digest)) + if not rows: + raise InstalledWheelProofError("retained wheel has no immutable payload members") + return len(rows), _sha256_bytes("".join(rows).encode("utf-8")) + + +def _installed_distribution_paths( + distribution: importlib.metadata.Distribution, +) -> tuple[Path, Path, Path]: + """Resolve package/native paths from distribution metadata without importing PoPS.""" + + members = tuple(distribution.files or ()) + package_members = [ + member for member in members if member.as_posix() == "pops/__init__.py" + ] + native_members = [ + member + for member in members + if member.parent.as_posix() == "pops" + and member.name.startswith("_pops.") + and any(member.name.endswith(suffix) for suffix in importlib.machinery.EXTENSION_SUFFIXES) + ] + if len(package_members) != 1 or len(native_members) != 1: + raise InstalledWheelProofError( + "installed distribution lacks one unique pops package and native extension" + ) + return ( + Path(distribution.locate_file(package_members[0])).resolve(), + Path(distribution.locate_file(native_members[0])).resolve(), + Path(distribution.locate_file("")).resolve(), + ) + + def build_proof( wheel: Path, *, @@ -114,6 +180,10 @@ def build_proof( native_member = native_members[0] native_digest = _sha256_bytes(archive.read(native_member)) metadata = archive.read(metadata_members[0]).decode("utf-8") + installed_member_count, installed_tree_sha256 = _wheel_payload_proof( + archive, + distribution_root=distribution, + ) except (OSError, UnicodeDecodeError, zipfile.BadZipFile) as exc: raise InstalledWheelProofError("retained wheel is unreadable: %s" % exc) from exc @@ -143,6 +213,9 @@ def build_proof( "native_extension": str(extension), "native_member": native_member, "native_sha256": native_digest, + "installed_member_count": installed_member_count, + "installed_tree_sha256": installed_tree_sha256, + "proof_script_sha256": _sha256(Path(__file__).resolve()), "version": installed_version, "wheel_path": str(retained), "wheel_sha256": wheel_digest, @@ -152,10 +225,10 @@ def build_proof( def installed_wheel_proof(wheel: Path) -> dict[str, Any]: """Resolve the live imported distribution and authenticate it against ``wheel``.""" - import pops - from pops import _pops - distribution = importlib.metadata.distribution("pops") + package_file, native_extension, distribution_root = _installed_distribution_paths( + distribution + ) direct_url_text = distribution.read_text("direct_url.json") if direct_url_text is None: raise InstalledWheelProofError( @@ -165,13 +238,11 @@ def installed_wheel_proof(wheel: Path) -> dict[str, Any]: direct_url = json.loads(direct_url_text) except json.JSONDecodeError as exc: raise InstalledWheelProofError("installed direct_url.json is invalid JSON") from exc - if pops.__version__ != _pops.__version__ or pops.__version__ != distribution.version: - raise InstalledWheelProofError("installed Python/native/distribution versions disagree") return build_proof( wheel, - package_file=Path(pops.__file__), - native_extension=Path(_pops.__file__), - distribution_root=Path(distribution.locate_file("")), + package_file=package_file, + native_extension=native_extension, + distribution_root=distribution_root, python_executable=Path(sys.executable), installed_version=distribution.version, direct_url=direct_url, diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index 5ff58b793..57820f96c 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -254,13 +254,16 @@ def _installed_wheel_evidence( "native_extension", "native_member", "native_sha256", + "installed_member_count", + "installed_tree_sha256", + "proof_script_sha256", "version", "wheel_path", "wheel_sha256", } if not isinstance(evidence, dict) or set(evidence) != expected: raise PreflightError("installed wheel evidence is malformed") - if evidence["schema_version"] != 1: + if evidence["schema_version"] != 2: raise PreflightError("installed wheel evidence schema is unsupported") if evidence["version"] != contract.PACKAGE_VERSION: raise PreflightError("installed wheel evidence version disagrees with release contract") @@ -298,10 +301,27 @@ def _installed_wheel_evidence( with zipfile.ZipFile(retained) as archive: member = evidence["native_member"] member_digest = hashlib.sha256(archive.read(member)).hexdigest() + rows = [] + for name in sorted(archive.namelist()): + if name.endswith("/") or name.endswith(".dist-info/RECORD"): + continue + if ".data/" in name: + raise PreflightError( + "release wheel uses an unsupported .data installation scheme" + ) + digest = hashlib.sha256(archive.read(name)).hexdigest() + rows.append("%s\0%s\n" % (name, digest)) except (KeyError, OSError, zipfile.BadZipFile) as exc: raise PreflightError("installed wheel native member is unreadable: %s" % exc) from exc if member_digest != evidence["native_sha256"]: raise PreflightError("installed wheel native member hash drifted") + expected_tree = hashlib.sha256("".join(rows).encode("utf-8")).hexdigest() + if evidence["installed_member_count"] != len(rows) \ + or evidence["installed_tree_sha256"] != expected_tree: + raise PreflightError("installed wheel payload proof drifted") + proof_script = ROOT / "scripts" / "prove_installed_wheel.py" + if evidence["proof_script_sha256"] != hashlib.sha256(proof_script.read_bytes()).hexdigest(): + raise PreflightError("installed wheel proof script drifted") def _codesign_evidence( diff --git a/tests/python/architecture/test_final_release_gate.py b/tests/python/architecture/test_final_release_gate.py index e23a5dc6d..4f5909843 100644 --- a/tests/python/architecture/test_final_release_gate.py +++ b/tests/python/architecture/test_final_release_gate.py @@ -200,6 +200,7 @@ def test_installed_wheel_proof_requires_exact_native_member_and_direct_url(tmp_p wheel = tmp_path / "pops-0.3.0-cp312-cp312-macosx_11_0_arm64.whl" native_bytes = b"exact wheel extension" with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("pops/__init__.py", "__version__ = '0.3.0'\n") archive.writestr("pops/_pops.cpython-312-darwin.so", native_bytes) archive.writestr( "pops-0.3.0.dist-info/METADATA", @@ -211,6 +212,12 @@ def test_installed_wheel_proof_requires_exact_native_member_and_direct_url(tmp_p package.parent.mkdir(parents=True) package.write_text("__version__ = '0.3.0'\n", encoding="utf-8") extension.write_bytes(native_bytes) + metadata = distribution / "pops-0.3.0.dist-info" / "METADATA" + metadata.parent.mkdir() + metadata.write_text( + "Metadata-Version: 2.3\nName: PoPS\nVersion: 0.3.0\n", + encoding="utf-8", + ) wheel_sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() direct_url = { "archive_info": {"hashes": {"sha256": wheel_sha256}}, @@ -229,6 +236,7 @@ def test_installed_wheel_proof_requires_exact_native_member_and_direct_url(tmp_p assert proof["wheel_sha256"] == wheel_sha256 assert proof["native_sha256"] == hashlib.sha256(native_bytes).hexdigest() + assert proof["installed_member_count"] == 3 extension.write_bytes(b"not the retained wheel") with pytest.raises(installed.InstalledWheelProofError, match="not byte-identical"): installed.build_proof( @@ -242,12 +250,24 @@ def test_installed_wheel_proof_requires_exact_native_member_and_direct_url(tmp_p ) +def test_installed_wheel_resolver_never_imports_unsigned_native_extension() -> None: + source = (SCRIPTS / "prove_installed_wheel.py").read_text(encoding="utf-8") + resolver = source.split("def installed_wheel_proof(", 1)[1].split( + "\ndef main(", 1 + )[0] + + assert "import pops" not in resolver + assert "from pops import" not in resolver + assert "_installed_distribution_paths(" in resolver + + def test_release_preflight_authenticates_installed_wheel_proof_and_transcripts(tmp_path): wheel = tmp_path / "wheels" / "pops-0.3.0-cp312-cp312-macosx_11_0_arm64.whl" wheel.parent.mkdir() native_member = "pops/_pops.cpython-312-darwin.so" native_bytes = b"exact wheel extension" with zipfile.ZipFile(wheel, "w") as archive: + archive.writestr("pops/__init__.py", "__version__ = '0.3.0'\n") archive.writestr(native_member, native_bytes) archive.writestr( "pops-0.3.0.dist-info/METADATA", @@ -260,6 +280,13 @@ def test_release_preflight_authenticates_installed_wheel_proof_and_transcripts(t "native_sha256": "post-sign-runtime-digest", } wheel_sha256 = hashlib.sha256(wheel.read_bytes()).hexdigest() + with zipfile.ZipFile(wheel) as archive: + rows = [ + "%s\0%s\n" + % (name, hashlib.sha256(archive.read(name)).hexdigest()) + for name in sorted(archive.namelist()) + if not name.endswith("/") and not name.endswith(".dist-info/RECORD") + ] commands = [] command_argvs = ( [ @@ -306,13 +333,20 @@ def test_release_preflight_authenticates_installed_wheel_proof_and_transcripts(t "installed_wheel": { "commands": commands, "evidence": { - "schema_version": 1, + "schema_version": 2, "python_executable": runtime["python_executable"], "distribution_root": "/proof/site-packages", "package_file": runtime["pops_file"], "native_extension": runtime["native_extension"], "native_member": native_member, "native_sha256": hashlib.sha256(native_bytes).hexdigest(), + "installed_member_count": len(rows), + "installed_tree_sha256": hashlib.sha256( + "".join(rows).encode("utf-8") + ).hexdigest(), + "proof_script_sha256": hashlib.sha256( + (SCRIPTS / "prove_installed_wheel.py").read_bytes() + ).hexdigest(), "version": "0.3.0", "wheel_path": str(wheel), "wheel_sha256": wheel_sha256, From 6300ebdb47e92058648bdc31c65a6889a2c731ea Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:13:03 +0200 Subject: [PATCH 056/109] api: prove source and wheel public parity --- scripts/prove_public_api_parity.py | 338 ++++++++++++++++++ .../test_public_api_parity_proof.py | 77 ++++ 2 files changed, 415 insertions(+) create mode 100644 scripts/prove_public_api_parity.py create mode 100644 tests/python/architecture/test_public_api_parity_proof.py diff --git a/scripts/prove_public_api_parity.py b/scripts/prove_public_api_parity.py new file mode 100644 index 000000000..013806506 --- /dev/null +++ b/scripts/prove_public_api_parity.py @@ -0,0 +1,338 @@ +#!/usr/bin/env python3 +"""Prove that the release wheel and source checkout expose one pure-Python API.""" + +from __future__ import annotations + +import argparse +from collections.abc import Mapping, Sequence +import hashlib +import json +from pathlib import Path, PurePosixPath +import subprocess +import sys +import tempfile +from typing import Any +import zipfile + + +ROOT = Path(__file__).resolve().parents[1] +SOURCE_PACKAGE = ROOT / "python" / "pops" +PROOF_SCHEMA_VERSION = 1 +TYPED_PAYLOAD_SUFFIXES = (".py", ".pyi") +PUBLIC_ROOT = ( + "Model", + "Program", + "Case", + "RunReport", + "RunStopReason", + "ExecutionContext", + "set_threads", + "validate", + "inspect", + "explain", + "resolve", + "compile", + "bind", + "run", + "__version__", +) + +_SNAPSHOT_PROGRAM = r""" +import hashlib +import inspect as _inspect +import json +from pathlib import Path +import sys + +package_parent = Path(sys.argv[1]).resolve() +sys.path.insert(0, str(package_parent)) +import pops + +expected_retired = ( + "Problem", + "RuntimePolicies", + "OutputPolicy", + "CheckpointPolicy", + "System", + "AmrSystem", + "ModelSpec", + "BindInputs", + "SystemConfig", + "AmrSystemConfig", + "CompiledTime", + "compile_library", + "read_library_manifest", + "LibraryManifest", +) +expected_public = ( + "Model", + "Program", + "Case", + "RunReport", + "RunStopReason", + "ExecutionContext", + "set_threads", + "validate", + "inspect", + "explain", + "resolve", + "compile", + "bind", + "run", + "__version__", +) +if tuple(pops.__all__) != expected_public: + raise RuntimeError("root public API does not match the final contract") +if "pops._pops" in sys.modules: + raise RuntimeError("root import loaded pops._pops") +if not isinstance(pops.Case, type) or "__getattr__" in pops.Case.__dict__: + raise RuntimeError("Case is not one explicit public type") +if "__getattr__" in pops.__dict__: + raise RuntimeError("root package uses a dynamic public facade") +if any(hasattr(pops, name) for name in expected_retired): + raise RuntimeError("root package still exposes a replaced public name") +if not (Path(pops.__file__).resolve().parent / "py.typed").is_file(): + raise RuntimeError("package has no py.typed marker") + +model = pops.Model("parity") +state = model.state("U", components=("u",)) +case = pops.Case("two_instances") +left = case.block("left", model) +right = case.block("right", model) +left_state = case.qualify(state, block=left) +right_state = case.qualify(state, block=right) +if left_state == right_state or left_state.block_ref != left or right_state.block_ref != right: + raise RuntimeError("qualified handles do not disambiguate repeated Model instances") +if pops.validate(case) is not case or not case.frozen: + raise RuntimeError("pure-Python validation did not freeze the exact Case") +report = pops.inspect(case) +if report["name"] != "two_instances" or set(report["blocks"]) != {"left", "right"}: + raise RuntimeError("pure-Python inspection did not preserve qualified blocks") +if "pops._pops" in sys.modules: + raise RuntimeError("authoring, validation, or inspection loaded pops._pops") + +def _annotation(value): + if isinstance(value, str): + return value + module = getattr(value, "__module__", None) + qualname = getattr(value, "__qualname__", None) + if module and qualname: + return module + "." + qualname + return repr(value) + +def _symbol(name): + value = getattr(pops, name) + if _inspect.isclass(value): + kind = "class" + elif _inspect.isfunction(value): + kind = "function" + else: + kind = type(value).__name__ + try: + call_signature = str(_inspect.signature(value, eval_str=False)) + except (TypeError, ValueError): + call_signature = None + annotations = getattr(value, "__annotations__", {}) + return { + "kind": kind, + "module": getattr(value, "__module__", None), + "qualname": getattr(value, "__qualname__", None), + "signature": call_signature, + "annotations": { + key: _annotation(annotation) + for key, annotation in sorted(annotations.items()) + }, + } + +public = list(pops.__all__) +snapshot = { + "public": public, + "symbols": {name: _symbol(name) for name in public}, + "case_is_explicit_type": True, + "qualified_handles": True, + "pure_authoring": True, + "py_typed": True, +} +print(json.dumps(snapshot, sort_keys=True, separators=(",", ":"))) +""" + + +class PublicApiParityError(RuntimeError): + """The source checkout and release wheel do not expose one exact public API.""" + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _sha256(path: Path) -> str: + return _sha256_bytes(path.read_bytes()) + + +def _is_typed_payload(relative: str) -> bool: + path = PurePosixPath(relative) + return path.name == "py.typed" or path.suffix in TYPED_PAYLOAD_SUFFIXES + + +def _source_manifest(package: Path = SOURCE_PACKAGE) -> dict[str, str]: + if not package.is_dir(): + raise PublicApiParityError("source package is absent: %s" % package) + manifest = { + path.relative_to(package).as_posix(): _sha256(path) + for path in sorted(package.rglob("*")) + if path.is_file() + and "__pycache__" not in path.parts + and _is_typed_payload(path.relative_to(package).as_posix()) + } + required = {"__init__.py", "_pops.pyi", "py.typed"} + if not required.issubset(manifest): + raise PublicApiParityError("source package lacks its root API or typing payload") + return manifest + + +def _wheel_manifest(archive: zipfile.ZipFile) -> dict[str, str]: + members = [ + info + for info in archive.infolist() + if not info.is_dir() and info.filename.startswith("pops/") + ] + names = [info.filename for info in members] + if len(names) != len(set(names)): + raise PublicApiParityError("release wheel contains duplicate pops package members") + manifest = { + info.filename.removeprefix("pops/"): _sha256_bytes(archive.read(info)) + for info in members + if _is_typed_payload(info.filename.removeprefix("pops/")) + } + required = {"__init__.py", "_pops.pyi", "py.typed"} + if not required.issubset(manifest): + raise PublicApiParityError("release wheel lacks its root API or typing payload") + return manifest + + +def _safe_extract(archive: zipfile.ZipFile, destination: Path) -> None: + for info in archive.infolist(): + relative = PurePosixPath(info.filename) + if relative.is_absolute() or ".." in relative.parts: + raise PublicApiParityError("release wheel contains an unsafe member path") + archive.extractall(destination) + + +def _snapshot(package_parent: Path) -> dict[str, Any]: + completed = subprocess.run( + [sys.executable, "-I", "-c", _SNAPSHOT_PROGRAM, str(package_parent.resolve())], + cwd=ROOT, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + env={"PYTHONDONTWRITEBYTECODE": "1"}, + ) + if completed.returncode: + raise PublicApiParityError( + "public API snapshot failed for %s:\n%s" + % (package_parent, completed.stdout[-4000:]) + ) + try: + payload = json.loads(completed.stdout) + except json.JSONDecodeError as exc: + raise PublicApiParityError( + "public API snapshot was not JSON for %s" % package_parent + ) from exc + if not isinstance(payload, dict): + raise PublicApiParityError("public API snapshot is not an object") + return payload + + +def _canonical_sha256(payload: Mapping[str, Any]) -> str: + encoded = json.dumps( + payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True + ).encode("utf-8") + return _sha256_bytes(encoded) + + +def build_proof(wheel: Path) -> dict[str, Any]: + """Compare one exact wheel archive with the current source checkout.""" + retained = wheel.expanduser().resolve() + if retained.suffix != ".whl" or not retained.is_file(): + raise PublicApiParityError("release artifact is not one readable wheel") + source_manifest = _source_manifest() + try: + with tempfile.TemporaryDirectory(prefix="pops-public-api-") as temporary: + extracted = Path(temporary) + with zipfile.ZipFile(retained) as archive: + wheel_manifest = _wheel_manifest(archive) + if wheel_manifest != source_manifest: + missing = sorted(set(source_manifest) - set(wheel_manifest)) + extra = sorted(set(wheel_manifest) - set(source_manifest)) + changed = sorted( + name + for name in set(source_manifest) & set(wheel_manifest) + if source_manifest[name] != wheel_manifest[name] + ) + raise PublicApiParityError( + "wheel Python/typing payload differs from source " + "(missing=%s, extra=%s, changed=%s)" + % (missing[:8], extra[:8], changed[:8]) + ) + _safe_extract(archive, extracted) + source_snapshot = _snapshot(SOURCE_PACKAGE.parent) + wheel_snapshot = _snapshot(extracted) + except (OSError, zipfile.BadZipFile) as exc: + raise PublicApiParityError("release wheel is unreadable: %s" % exc) from exc + if wheel_snapshot != source_snapshot: + raise PublicApiParityError("wheel and source public API snapshots differ") + if tuple(source_snapshot["public"]) != PUBLIC_ROOT: + raise PublicApiParityError("public API snapshot differs from the final root contract") + return { + "schema_version": PROOF_SCHEMA_VERSION, + "wheel_path": str(retained), + "wheel_sha256": _sha256(retained), + "typed_payload_files": len(source_manifest), + "typed_payload_sha256": _canonical_sha256(source_manifest), + "public_api_sha256": _canonical_sha256(source_snapshot), + "public_names": source_snapshot["public"], + "pure_authoring": source_snapshot["pure_authoring"], + "qualified_handles": source_snapshot["qualified_handles"], + "py_typed": source_snapshot["py_typed"], + } + + +def _write_evidence(path: Path, proof: Mapping[str, Any]) -> None: + destination = path.expanduser().resolve() + try: + destination.relative_to(ROOT) + except ValueError: + pass + else: + raise PublicApiParityError("evidence path must be outside the checkout") + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.exists(): + raise PublicApiParityError("refusing to overwrite public API evidence: %s" % destination) + with tempfile.NamedTemporaryFile( + "w", encoding="utf-8", dir=destination.parent, delete=False + ) as stream: + json.dump(proof, stream, sort_keys=True, indent=2) + stream.write("\n") + temporary = Path(stream.name) + temporary.replace(destination) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--wheel", required=True, type=Path) + parser.add_argument("--evidence", type=Path) + args = parser.parse_args(argv) + try: + proof = build_proof(args.wheel) + if args.evidence is not None: + _write_evidence(args.evidence, proof) + except (PublicApiParityError, OSError, ValueError) as exc: + print("public API parity proof failed: %s" % exc, file=sys.stderr) + return 1 + print(json.dumps(proof, sort_keys=True)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/python/architecture/test_public_api_parity_proof.py b/tests/python/architecture/test_public_api_parity_proof.py new file mode 100644 index 000000000..53af6f9c8 --- /dev/null +++ b/tests/python/architecture/test_public_api_parity_proof.py @@ -0,0 +1,77 @@ +"""ADC-689 source/wheel public API and typing parity proof.""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +import sys +import zipfile + +import pytest + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "prove_public_api_parity.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("_public_api_parity_test", SCRIPT) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +proof = _load() + + +def _synthetic_wheel(path: Path, *, omit: str | None = None) -> None: + with zipfile.ZipFile(path, "w") as archive: + for source in sorted(proof.SOURCE_PACKAGE.rglob("*")): + if not source.is_file() or "__pycache__" in source.parts: + continue + relative = source.relative_to(proof.SOURCE_PACKAGE).as_posix() + if relative == omit: + continue + archive.write(source, "pops/" + relative) + archive.writestr( + "pops-1.0.0.dist-info/METADATA", + "Metadata-Version: 2.3\nName: PoPS\nVersion: 1.0.0\n", + ) + + +def test_exact_wheel_and_source_share_public_api_typing_and_lazy_authoring(tmp_path): + wheel = tmp_path / "pops-1.0.0-py3-none-any.whl" + _synthetic_wheel(wheel) + + evidence = proof.build_proof(wheel) + + assert evidence["schema_version"] == 1 + assert evidence["public_names"] == list(proof.PUBLIC_ROOT) + assert evidence["pure_authoring"] is True + assert evidence["qualified_handles"] is True + assert evidence["py_typed"] is True + assert evidence["typed_payload_files"] > 100 + + +def test_wheel_proof_fails_closed_when_typing_payload_is_missing(tmp_path): + wheel = tmp_path / "pops-1.0.0-py3-none-any.whl" + _synthetic_wheel(wheel, omit="_pops.pyi") + + with pytest.raises(proof.PublicApiParityError, match="typing payload"): + proof.build_proof(wheel) + + +def test_release_workflow_blocks_publication_on_source_wheel_api_parity(): + workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text( + encoding="utf-8" + ) + validate = workflow[workflow.index(" validate:") : workflow.index(" release:")] + + assert "scripts/prove_public_api_parity.py" in validate + assert '--wheel "${wheels[0]}"' in validate + assert 'pops-final-evidence-public-api.json' in validate + assert validate.index("scripts/prove_public_api_parity.py") < validate.index( + "scripts/run_final_gate.py" + ) From 23d9aca752894e7ab1f62f86975df4d5ea7d348e Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:13:19 +0200 Subject: [PATCH 057/109] release: gate publication on public API parity --- .github/workflows/release.yml | 3 +++ .../SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md | 9 +++++++++ docs/docmap.toml | 3 +++ 3 files changed, 15 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2c2714279..5ebbba136 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,6 +61,9 @@ jobs: wheels=("$RUNNER_TEMP"/wheelhouse/pops-*.whl) test "${#wheels[@]}" -eq 1 evidence="$RUNNER_TEMP/pops-final-evidence.json" + python scripts/prove_public_api_parity.py \ + --wheel "${wheels[0]}" \ + --evidence "$RUNNER_TEMP/pops-final-evidence-public-api.json" python scripts/run_final_gate.py --wheel "${wheels[0]}" --evidence "$evidence" python - <<'PY' from pops.runtime_environment import runtime_environment_report diff --git a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md index e8bb1baf1..5633c7c83 100644 --- a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md +++ b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md @@ -1536,6 +1536,15 @@ dans `examples/final/`. Chaque script doit : ## 14. Gate de conformance finale +Le job de release commence par +`scripts/prove_public_api_parity.py --wheel --evidence `. +Cette preuve compare octet par octet tous les fichiers Python et de typage (`*.py`, `*.pyi`, +`py.typed`) du checkout et du wheel retenu, puis importe séparément les deux arbres dans des +interpréteurs isolés. Les deux snapshots doivent exposer la même racine publique, les mêmes +signatures et annotations, un `Case` explicite, des handles qualifiés distincts et +authoring/validation/inspection sans chargement de `_pops`. Un ancien nom public, un fichier de +typage absent ou une divergence source/wheel bloque la publication. + Une release ne peut être déclarée conforme que par `scripts/run_final_gate.py --evidence `. La commande exige un checkout propre, refuse d'écraser une evidence existante et produit une evidence JSON liée au commit, à la version du diff --git a/docs/docmap.toml b/docs/docmap.toml index 812be7ecc..c5663ad22 100644 --- a/docs/docmap.toml +++ b/docs/docmap.toml @@ -84,6 +84,8 @@ depends_on = [ "python/pops/physics/board.py", "python/pops/problem/problem.py", "python/pops/time/_program/api.py", + "scripts/prove_public_api_parity.py", + ".github/workflows/release.yml", "examples/final/EXEMPLE_SPEC_FINALE_ADVECTION_SCALAIRE_COMPLET.py", "examples/final/EXEMPLE_SPEC_FINALE_MULTIPHYSIQUE_CORE.py", "examples/final/EXEMPLE_SPEC_FINALE_ADVECTION_IMEX_AMR.py", @@ -91,6 +93,7 @@ depends_on = [ ] tested_by = [ "tests/python/architecture/test_final_public_api.py", + "tests/python/architecture/test_public_api_parity_proof.py", "tests/python/architecture/test_release_contract.py", ] testable = true From 8b6133cc8b29cf37f45e4b604f0285bd443e170e Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:33:52 +0200 Subject: [PATCH 058/109] feat(amr): execute prepared local Reflux kernels --- .../time/amr/levels/amr_patch_range.hpp | 92 ++++++++ .../time/amr/levels/amr_subcycling.hpp | 217 ++++++++++++++++-- .../pops/runtime/amr/amr_program_reflux.hpp | 5 +- .../runtime/program/amr_program_context.hpp | 2 +- 4 files changed, 300 insertions(+), 16 deletions(-) diff --git a/include/pops/numerics/time/amr/levels/amr_patch_range.hpp b/include/pops/numerics/time/amr/levels/amr_patch_range.hpp index 02a52a46b..49d4d497a 100644 --- a/include/pops/numerics/time/amr/levels/amr_patch_range.hpp +++ b/include/pops/numerics/time/amr/levels/amr_patch_range.hpp @@ -593,6 +593,29 @@ struct RefluxStripConstView { int components = 0; }; +/// Four already-computed, signed coarse-cell corrections around one fine-patch footprint. +/// +/// A native Reflux component may fill these contiguous face buffers, but it never receives the +/// sparse global register, coverage mask, periodicity or MPI communicator. PoPS alone maps the +/// values onto canonical uncovered parent cells. +struct RefluxFaceCorrectionView { + int I0 = 0, I1 = -1, J0 = 0, J1 = -1; + Real* x_low = nullptr; + Real* x_high = nullptr; + Real* y_low = nullptr; + Real* y_high = nullptr; + int components = 0; +}; + +struct RefluxFaceCorrectionConstView { + int I0 = 0, I1 = -1, J0 = 0, J1 = -1; + const Real* x_low = nullptr; + const Real* x_high = nullptr; + const Real* y_low = nullptr; + const Real* y_high = nullptr; + int components = 0; +}; + template inline RefluxStripView reflux_strip_view(Strip& strip, int components) { return {strip.I0, strip.I1, strip.J0, strip.J1, strip.cL.data(), @@ -821,6 +844,64 @@ struct RouteRefluxStripKernel { } }; +/// Deposit one external/local Reflux result into PoPS' sparse correction authority. The provider +/// has already applied side*(fine-coarse)/spacing; this kernel owns only topology canonicalisation, +/// coverage exclusion and deterministic face order. +struct RoutePreparedRefluxCorrectionKernel { + RefluxFaceCorrectionConstView faces; + FluxRegisterView correction; + CoverageMaskView coverage; + Box2D coarse_domain; + Periodicity periodicity; + + POPS_HD static int wrap_index(int value, int lo, int extent) { + const std::int64_t relative = static_cast(value) - lo; + std::int64_t quotient = relative / extent; + if (relative % extent < 0) + --quotient; + return static_cast(static_cast(lo) + relative - quotient * extent); + } + + POPS_HD bool canonicalize(int& I, int& J) const { + if (I < coarse_domain.lo[0] || I > coarse_domain.hi[0]) { + if (!periodicity.x) + return false; + I = wrap_index(I, coarse_domain.lo[0], coarse_domain.nx()); + } + if (J < coarse_domain.lo[1] || J > coarse_domain.hi[1]) { + if (!periodicity.y) + return false; + J = wrap_index(J, coarse_domain.lo[1], coarse_domain.ny()); + } + return true; + } + + POPS_HD void add_if_uncovered(int I, int J, int component, Real amount) const { + if (!canonicalize(I, J) || coverage.covered(I, J)) + return; + correction.add(I, J, component, amount); + } + + POPS_HD void operator()(int, int) const { + for (int J = faces.J0; J <= faces.J1; ++J) + for (int component = 0; component < faces.components; ++component) { + const std::size_t index = + static_cast(J - faces.J0) * static_cast(faces.components) + + static_cast(component); + add_if_uncovered(faces.I0 - 1, J, component, faces.x_low[index]); + add_if_uncovered(faces.I1 + 1, J, component, faces.x_high[index]); + } + for (int I = faces.I0; I <= faces.I1; ++I) + for (int component = 0; component < faces.components; ++component) { + const std::size_t index = + static_cast(I - faces.I0) * static_cast(faces.components) + + static_cast(component); + add_if_uncovered(I, faces.J0 - 1, component, faces.y_low[index]); + add_if_uncovered(I, faces.J1 + 1, component, faces.y_high[index]); + } + } +}; + } // namespace detail inline void sample_coarse_x_strip(const ConstArray4& left, const ConstArray4& right, @@ -1087,6 +1168,17 @@ struct CoarseFineInterface { Real(1) / dx, Real(1) / dy, Real(1)}); } + void route_prepared_reflux_correction_(const RefluxFaceCorrectionConstView& faces, + FluxRegister& ref, int nc) const { + if (nc <= 0 || ref.nc != nc || faces.components != nc || faces.I1 < faces.I0 || + faces.J1 < faces.J0 || faces.x_low == nullptr || faces.x_high == nullptr || + faces.y_low == nullptr || faces.y_high == nullptr) + throw std::invalid_argument("prepared Reflux correction view is incomplete"); + for_each_cell(Box2D{{0, 0}, {0, 0}}, + detail::RoutePreparedRefluxCorrectionKernel{faces, ref.view(), cmask.view(), + coarse_region, periodicity}); + } + template static void validate_route_inputs_(const Reg& coarse, const Reg& fine, Real dx, Real dy, Real coarse_scale, const FluxRegister& ref, int nc, diff --git a/include/pops/numerics/time/amr/levels/amr_subcycling.hpp b/include/pops/numerics/time/amr/levels/amr_subcycling.hpp index 12292338a..ac48aaece 100644 --- a/include/pops/numerics/time/amr/levels/amr_subcycling.hpp +++ b/include/pops/numerics/time/amr/levels/amr_subcycling.hpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -796,6 +797,90 @@ inline void clear_reflux_storage_on_device(RefluxStorage& values) { } // namespace detail +/// Persistent, patch-local output storage for one external Reflux invocation. It is allocated with +/// the topology plan, poisoned before every callback and consumed by PoPS only after all entries are +/// finite. Non-owning ABI views never outlive this workspace. +struct PreparedAmrRefluxFaceWorkspace { + int I0 = 0, I1 = -1, J0 = 0, J1 = -1; + int components = 0; + RefluxStorage x_low; + RefluxStorage x_high; + RefluxStorage y_low; + RefluxStorage y_high; + std::string patch_identity; + std::array interface_identities; + + static PreparedAmrRefluxFaceWorkspace prepare(const Box2D& footprint, int ncomp, + std::string transition_identity, + std::size_t global_child) { + if (footprint.empty() || ncomp <= 0 || transition_identity.empty()) + throw std::invalid_argument("prepared external Reflux workspace is incomplete"); + const auto checked_size = [ncomp](std::int64_t extent) { + const std::size_t components = static_cast(ncomp); + if (extent <= 0 || + static_cast(extent) > std::numeric_limits::max() / components) + throw std::overflow_error("prepared external Reflux face size overflow"); + return static_cast(extent) * components; + }; + PreparedAmrRefluxFaceWorkspace result; + result.I0 = footprint.lo[0]; + result.I1 = footprint.hi[0]; + result.J0 = footprint.lo[1]; + result.J1 = footprint.hi[1]; + result.components = ncomp; + result.x_low.resize(checked_size(footprint.ny())); + result.x_high.resize(result.x_low.size()); + result.y_low.resize(checked_size(footprint.nx())); + result.y_high.resize(result.y_low.size()); + result.patch_identity = transition_identity + "/patch=" + std::to_string(global_child); + result.interface_identities = { + result.patch_identity + "/x-low", result.patch_identity + "/x-high", + result.patch_identity + "/y-low", result.patch_identity + "/y-high"}; + return result; + } + + void poison() { + const Real sentinel = std::numeric_limits::quiet_NaN(); + for (auto* values : {&x_low, &x_high, &y_low, &y_high}) + std::fill(values->begin(), values->end(), sentinel); + } + + [[nodiscard]] bool all_finite() const { + for (const auto* values : {&x_low, &x_high, &y_low, &y_high}) + if (std::any_of(values->begin(), values->end(), + [](Real value) { return !std::isfinite(static_cast(value)); })) + return false; + return true; + } + + [[nodiscard]] RefluxFaceCorrectionView view() { + return {I0, I1, J0, J1, x_low.data(), x_high.data(), y_low.data(), y_high.data(), components}; + } + + [[nodiscard]] RefluxFaceCorrectionConstView view() const { + return {I0, I1, J0, J1, x_low.data(), x_high.data(), y_low.data(), y_high.data(), components}; + } +}; + +/// Complete local/noncollective invocation data. Flux strips are already integrated in time and +/// averaged onto coarse faces; the callback may only fill `correction`. +struct PreparedAmrRefluxLocalRequest { + const std::string* transition_identity = nullptr; + const std::string* patch_identity = nullptr; + std::array interface_identities{}; + int parent_level = -1; + int child_level = -1; + std::size_t global_child = 0; + RefluxStripConstView coarse; + RefluxStripConstView fine; + RefluxFaceCorrectionView correction; + amr::ClockStamp logical_time; + Real dx = Real(0); + Real dy = Real(0); +}; + +using PreparedAmrRefluxLocalKernel = std::function; + /// Prepared spatial reflux storage for one exact Program-owned parent/child transition. It owns /// only the interface topology and collective correction register; ProgramGraph supplies the /// already time-integrated coarse/fine flux strips. @@ -812,19 +897,42 @@ class PreparedAmrProgramRefluxTransition { const Box2D& parent_domain, Periodicity periodicity, const CommunicatorView& communicator) { + return prepare_with_local_kernel(parent, child, parent_domain, periodicity, 0, + "pops://runtime/amr/program-reflux/parent=0/child=1", {}, + communicator); + } + + static PreparedAmrProgramRefluxTransition prepare_with_local_kernel( + const AmrLevelMP& parent, const AmrLevelMP& child, const Box2D& parent_domain, + Periodicity periodicity, int parent_level, std::string transition_identity, + PreparedAmrRefluxLocalKernel local_kernel, const CommunicatorView& communicator) { if (parent.U.ncomp() != child.U.ncomp()) throw std::invalid_argument("prepared AMR Program reflux transition component mismatch"); + if (parent_level < 0 || transition_identity.empty()) + throw std::invalid_argument("prepared AMR Program reflux transition identity is incomplete"); validate_ratio_aligned_disjoint_fine_layout(child.U.box_array(), &parent_domain); CoarseFineInterface interface(parent_domain, child.U.box_array(), periodicity); std::vector correction_regions = interface.reflux_register_regions(child.U.box_array()); - return PreparedAmrProgramRefluxTransition(parent, child, communicator, std::move(interface), - std::move(correction_regions)); + std::vector local_workspaces( + static_cast(child.U.box_array().size())); + if (local_kernel) + for (int global_child = 0; global_child < child.U.box_array().size(); ++global_child) + if (child.U.dmap()[global_child] == communicator.rank()) + local_workspaces[static_cast(global_child)] = + PreparedAmrRefluxFaceWorkspace::prepare( + PatchRange(child.U.box_array()[global_child]).box(), parent.U.ncomp(), + transition_identity, static_cast(global_child)); + return PreparedAmrProgramRefluxTransition(parent, child, communicator, parent_level, + std::move(transition_identity), + std::move(local_kernel), std::move(local_workspaces), + std::move(interface), std::move(correction_regions)); } template void synchronize_integrated(MultiFab& parent_state, Real dx, Real dy, const CoarseStripRange& coarse_role, const FineStripRange& fine_role, - const CommunicatorView& communicator) { + const CommunicatorView& communicator, + const amr::ClockStamp* logical_time = nullptr) { validate_communicator_(communicator); using CoarseStrip = typename CoarseStripRange::value_type; using FineStrip = typename FineStripRange::value_type; @@ -836,6 +944,11 @@ class PreparedAmrProgramRefluxTransition { // enter the correction Allreduce while its peer unwinds. std::exception_ptr local_failure; try { + if (local_kernel_ && + (logical_time == nullptr || logical_time->level != parent_level_ || + logical_time->macro_step < 0 || !std::isfinite(logical_time->physical_time))) + throw std::invalid_argument( + "prepared external Reflux requires the exact parent logical time"); validate_parent_state_(parent_state); if (coarse_role.size() != child_global_size_ || fine_role.size() != child_global_size_) throw std::runtime_error( @@ -863,15 +976,69 @@ class PreparedAmrProgramRefluxTransition { } catch (...) { local_failure = std::current_exception(); } - const std::uint64_t rejected = - all_reduce_max(local_failure ? std::uint64_t(1) : std::uint64_t(0), communicator); - if (rejected != 0) { + // Presence, rank-local preflight failure and the later execution branch are decided by one + // collective bitmask. A rank can therefore never enter the builtin gather while a peer invokes + // an external callback. + constexpr char kExternalSelected = char{1}; + constexpr char kBuiltinSelected = char{2}; + constexpr char kPreflightFailed = char{4}; + char preflight_consensus = local_kernel_ ? kExternalSelected : kBuiltinSelected; + if (local_failure) + preflight_consensus |= kPreflightFailed; + all_reduce_or_inplace(&preflight_consensus, std::size_t{1}, communicator); + const bool provider_mismatch = (preflight_consensus & kExternalSelected) != 0 && + (preflight_consensus & kBuiltinSelected) != 0; + if ((preflight_consensus & kPreflightFailed) != 0 || provider_mismatch) { if (local_failure) std::rethrow_exception(local_failure); - throw std::runtime_error("AMR Program reflux preflight failed on another communicator rank"); + throw std::runtime_error(provider_mismatch + ? "prepared Reflux provider differs between communicator ranks" + : "AMR Program reflux preflight failed on another " + "communicator rank"); } + const bool use_external = (preflight_consensus & kExternalSelected) != 0; - try { + if (use_external) { + std::exception_ptr local_failure; + try { + correction_.clear_on_device(); + device_fence(); + for (std::size_t global_child = 0; global_child < child_global_size_; ++global_child) { + const CoarseStrip& coarse = coarse_role[global_child]; + const FineStrip& fine = fine_role[global_child]; + if (!coarse_role_present_(coarse)) + continue; + PreparedAmrRefluxFaceWorkspace& workspace = local_workspaces_[global_child]; + workspace.poison(); + std::array interface_identities; + for (std::size_t face = 0; face < interface_identities.size(); ++face) + interface_identities[face] = &workspace.interface_identities[face]; + local_kernel_(PreparedAmrRefluxLocalRequest{ + &transition_identity_, &workspace.patch_identity, interface_identities, parent_level_, + parent_level_ + 1, global_child, reflux_strip_const_view(coarse, ncomp_), + reflux_strip_const_view(fine, ncomp_), workspace.view(), *logical_time, dx, dy}); + if (!workspace.all_finite()) + throw std::runtime_error( + "native Reflux component left a non-finite or unwritten correction"); + const PreparedAmrRefluxFaceWorkspace& completed = workspace; + interface_.route_prepared_reflux_correction_(completed.view(), correction_, ncomp_); + } + device_fence(); + } catch (...) { + local_failure = std::current_exception(); + try { + device_fence(); + } catch (...) { + } + } + const std::uint64_t rejected = + all_reduce_max(local_failure ? std::uint64_t(1) : std::uint64_t(0), communicator); + if (rejected != 0) { + if (local_failure) + std::rethrow_exception(local_failure); + throw std::runtime_error("native Reflux component failed on another communicator rank"); + } + } else { correction_.clear_on_device(); for (std::size_t global_child = 0; global_child < child_global_size_; ++global_child) { const CoarseStrip& coarse = coarse_role[global_child]; @@ -881,6 +1048,8 @@ class PreparedAmrProgramRefluxTransition { interface_.route_reflux_integrated_pair_prevalidated_(coarse, fine, dx, dy, correction_, ncomp_); } + } + try { correction_.gather(communicator); for (int local_parent = 0; local_parent < parent_state.local_size(); ++local_parent) for_each_cell(parent_state.box(local_parent), @@ -900,7 +1069,10 @@ class PreparedAmrProgramRefluxTransition { private: PreparedAmrProgramRefluxTransition(const AmrLevelMP& parent, const AmrLevelMP& child, - const CommunicatorView& communicator, + const CommunicatorView& communicator, int parent_level, + std::string transition_identity, + PreparedAmrRefluxLocalKernel local_kernel, + std::vector local_workspaces, CoarseFineInterface interface, std::vector correction_regions) : parent_boxes_(parent.U.box_array().boxes()), @@ -913,6 +1085,10 @@ class PreparedAmrProgramRefluxTransition { communicator_size_(communicator.size()), communicator_rank_(communicator.rank()), communicator_identity_(detail::parallel_copy_communicator_identity(communicator)), + parent_level_(parent_level), + transition_identity_(std::move(transition_identity)), + local_kernel_(std::move(local_kernel)), + local_workspaces_(std::move(local_workspaces)), interface_(std::move(interface)), correction_(std::move(correction_regions), ncomp_) { if (child_footprints_.size() != child_global_size_ || child_ranks_.size() != child_global_size_) @@ -922,6 +1098,10 @@ class PreparedAmrProgramRefluxTransition { if (owner < 0 || owner >= communicator_size_) throw std::invalid_argument( "prepared AMR Program reflux child owner lies outside the communicator"); + if (parent_level_ < 0 || transition_identity_.empty() || + local_workspaces_.size() != child_global_size_) + throw std::invalid_argument( + "prepared AMR Program reflux local-provider metadata is inconsistent"); } static std::vector make_child_footprints_(const BoxArray& child_boxes) { @@ -972,6 +1152,10 @@ class PreparedAmrProgramRefluxTransition { int communicator_size_ = 1; int communicator_rank_ = 0; std::int64_t communicator_identity_ = 0; + int parent_level_ = 0; + std::string transition_identity_; + PreparedAmrRefluxLocalKernel local_kernel_; + std::vector local_workspaces_; CoarseFineInterface interface_; FluxRegister correction_; }; @@ -988,16 +1172,23 @@ class PreparedAmrProgramRefluxPlan { static PreparedAmrProgramRefluxPlan prepare( const std::vector& levels, const Box2D& base_domain, Periodicity periodicity, std::uint64_t topology_generation, - const CommunicatorView& communicator = world_communicator_view()) { + const CommunicatorView& communicator = world_communicator_view(), + PreparedAmrRefluxLocalKernel local_kernel = {}, std::string block_identity = {}) { if (levels.empty() || base_domain.empty()) throw std::invalid_argument("prepared AMR Program reflux requires a non-empty hierarchy"); + if (local_kernel && block_identity.empty()) + throw std::invalid_argument("prepared external Reflux requires one qualified block identity"); std::vector transitions; transitions.reserve(levels.size() - 1); - for (std::size_t parent = 0; parent + 1 < levels.size(); ++parent) - transitions.push_back(PreparedAmrProgramRefluxTransition::prepare( + for (std::size_t parent = 0; parent + 1 < levels.size(); ++parent) { + const std::string transition_identity = + (block_identity.empty() ? "pops://runtime/amr/program-reflux" : block_identity) + + "/parent=" + std::to_string(parent) + "/child=" + std::to_string(parent + 1); + transitions.push_back(PreparedAmrProgramRefluxTransition::prepare_with_local_kernel( levels[parent], levels[parent + 1], amr_level_index_domain(base_domain, static_cast(parent)), periodicity, - communicator)); + static_cast(parent), transition_identity, local_kernel, communicator)); + } return PreparedAmrProgramRefluxPlan(static_cast(levels.size()), topology_generation, std::move(transitions)); } diff --git a/include/pops/runtime/amr/amr_program_reflux.hpp b/include/pops/runtime/amr/amr_program_reflux.hpp index 96ab1bd65..3f20266e4 100644 --- a/include/pops/runtime/amr/amr_program_reflux.hpp +++ b/include/pops/runtime/amr/amr_program_reflux.hpp @@ -525,14 +525,15 @@ inline void sample_fine_role_strip(const MultiFab& state, const MultiFab& Fx, co /// per (cell,direction) (ADC-636 ownership: each C/F face is owned by the rank holding the covering fine /// patch), so the gather is associativity-free -> distributed == replicated bit-for-bit. inline void route_reflux_program(AmrRuntime& eng, std::size_t b, int k, const EdgeFlux& coarse_role, - const EdgeFlux& fine_role) { + const EdgeFlux& fine_role, const amr::ClockStamp& logical_time) { MultiFab& Uc = eng.level_state(b, k - 1); // the PARENT (coarse) live state we correct const BoxArray child_ba = eng.level_state(b, k).box_array(); // GLOBAL level-k patches if (child_ba.size() == 0) return; const Geometry gc = eng.level_geom(k - 1); eng.prepared_reflux_transition(b, k).synchronize_integrated( - Uc, gc.dx(), gc.dy(), coarse_role.coarse, fine_role.fine, world_communicator_view()); + Uc, gc.dx(), gc.dy(), coarse_role.coarse, fine_role.fine, world_communicator_view(), + &logical_time); } } // namespace detail diff --git a/include/pops/runtime/program/amr_program_context.hpp b/include/pops/runtime/program/amr_program_context.hpp index fa53c479a..beaf3c441 100644 --- a/include/pops/runtime/program/amr_program_context.hpp +++ b/include/pops/runtime/program/amr_program_context.hpp @@ -1124,7 +1124,7 @@ class AmrProgramContext : public ProgramExecutionServices { throw std::runtime_error( "AMR conservative ledger contains only one side of a parent/child flux pair"); if (!coarse_role.empty()) - pops::detail::route_reflux_program(*eng_, sb, child, coarse_role, fine_role); + pops::detail::route_reflux_program(*eng_, sb, child, coarse_role, fine_role, sync_clock); } sync_report_.push_back({parent, child, b, SyncPhase::AverageDown, sync_clock}); eng_->average_down_level(sb, child); From cddd9dac58b3a7d8bfd544842735c30130fa2c2a Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:34:05 +0200 Subject: [PATCH 059/109] feat(components): install Reflux into AMR transitions --- include/pops/runtime/amr/amr_runtime.hpp | 72 +++++- .../amr/prepared_component_providers.hpp | 207 ++++++++++++++++++ include/pops/runtime/amr_system.hpp | 2 + python/bindings/core/init/init_amr.cpp | 21 ++ src/runtime/amr/amr_system.cpp | 16 ++ 5 files changed, 317 insertions(+), 1 deletion(-) diff --git a/include/pops/runtime/amr/amr_runtime.hpp b/include/pops/runtime/amr/amr_runtime.hpp index 0326b3e63..bfc751ae6 100644 --- a/include/pops/runtime/amr/amr_runtime.hpp +++ b/include/pops/runtime/amr/amr_runtime.hpp @@ -37,6 +37,7 @@ #include #include #include // n_ranks() / comm_active(): MPI message+reduction counts (Spec 5 criterion 43) +#include #include #include #include @@ -1526,6 +1527,65 @@ class AmrRuntime { external_clustering_ = std::move(provider); } + void install_external_reflux(std::shared_ptr provider) { + const CommunicatorView communicator = world_communicator_view(); + std::exception_ptr local_failure; + try { + if (external_reflux_configured_ || bootstrap_pending_) + throw std::runtime_error( + "AmrRuntime external Reflux must be configured exactly once before bootstrap"); + } catch (...) { + local_failure = std::current_exception(); + } + if (all_reduce_max(local_failure ? std::uint64_t{1} : std::uint64_t{0}, communicator) != 0) { + if (communicator.size() == 1 && local_failure) + std::rethrow_exception(local_failure); + throw std::runtime_error( + "AmrRuntime external Reflux configuration failed on another communicator rank"); + } + + struct OptionalRefluxSelection { + const runtime::amr::PreparedRefluxComponent* provider = nullptr; + explicit operator bool() const noexcept { return provider != nullptr; } + [[nodiscard]] std::string_view collective_contract() const noexcept { + return provider == nullptr ? std::string_view{} : provider->collective_contract(); + } + }; + require_prepared_provider_collective_consensus(OptionalRefluxSelection{provider.get()}); + external_reflux_configured_ = true; + if (!provider) + return; + + external_reflux_ = std::move(provider); + local_failure = nullptr; + try { + rematerialize_persistent_topology_resources_(topology_materialization_generation_); + } catch (...) { + local_failure = std::current_exception(); + } + if (all_reduce_max(local_failure ? std::uint64_t{1} : std::uint64_t{0}, communicator) == 0) + return; + + external_reflux_.reset(); + std::exception_ptr rollback_failure; + try { + rematerialize_persistent_topology_resources_(topology_materialization_generation_); + } catch (...) { + rollback_failure = std::current_exception(); + } + external_reflux_configured_ = false; + if (all_reduce_max(rollback_failure ? std::uint64_t{1} : std::uint64_t{0}, communicator) != 0) { + if (communicator.size() == 1 && rollback_failure) + std::rethrow_exception(rollback_failure); + throw std::runtime_error( + "AmrRuntime external Reflux rollback failed on another communicator rank"); + } + if (communicator.size() == 1 && local_failure) + std::rethrow_exception(local_failure); + throw std::runtime_error( + "AmrRuntime external Reflux preparation failed on another communicator rank"); + } + /// Inject the current Program evaluation coordinate used by external Tagger/boundary component /// calls. This is not an accepted clock: it is never read for cadence/restart, is absent from /// StepSnapshot, and is overwritten by AmrProgramContext at the exact tagger/regrid boundary. @@ -5757,6 +5817,8 @@ class AmrRuntime { cluster_{}; ///< ADC-616: Berger-Rigoutsos params; default {0.7,1,32} (bit-identical). std::shared_ptr external_tagger_; std::shared_ptr external_clustering_; + std::shared_ptr external_reflux_; + bool external_reflux_configured_ = false; std::shared_ptr clustering_provider_ = std::make_shared(ClusterParams{}); // Ephemeral Program evaluation metadata required by the prepared component ABI. These values @@ -5924,6 +5986,13 @@ class AmrRuntime { coarse_fine_spatial_candidate.reserve(blocks_.size()); average_down_candidate.reserve(blocks_.size()); program_reflux_candidate.reserve(blocks_.size()); + PreparedAmrRefluxLocalKernel external_reflux_kernel; + if (external_reflux_) { + const std::shared_ptr provider = external_reflux_; + external_reflux_kernel = [provider](const PreparedAmrRefluxLocalRequest& request) { + provider->apply(request); + }; + } for (std::size_t block_index = 0; block_index < blocks_.size(); ++block_index) { const AmrRuntimeBlock& block = blocks_[block_index]; const auto& authority = block_transfer_authorities_[block_index]; @@ -5953,7 +6022,8 @@ class AmrRuntime { average_down_candidate.push_back( PreparedAmrAverageDownPlan::prepare(*block.levels, generation)); program_reflux_candidate.push_back(PreparedAmrProgramRefluxPlan::prepare( - *block.levels, dom_, base_per_, generation, world_communicator_view())); + *block.levels, dom_, base_per_, generation, world_communicator_view(), + external_reflux_kernel, block.state_identity)); } auto tagging_candidate = make_tagging_execution_plan_(tagging_program_, generation); temporal_parent_workspaces_.swap(temporal_candidate); diff --git a/include/pops/runtime/amr/prepared_component_providers.hpp b/include/pops/runtime/amr/prepared_component_providers.hpp index 6a13b1580..9b9aa7cc8 100644 --- a/include/pops/runtime/amr/prepared_component_providers.hpp +++ b/include/pops/runtime/amr/prepared_component_providers.hpp @@ -3,7 +3,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -20,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -186,6 +189,16 @@ struct PreparedClusteringSpec { std::shared_ptr execution; }; +struct PreparedRefluxSpec { + std::string provider_identity; + std::string component_id; + std::string manifest_identity; + std::string layout_identity; + std::string clock_identity; + std::uint32_t interface_version = 1; + std::shared_ptr execution; +}; + /// Prepared external Tagger. One invocation per local patch sees every graph input as a qualified /// borrowed SoA view and evaluates the exact resolved graph program. Only four Boolean candidate /// bitmaps are reduced across ranks; state arrays are never packed or globally reduced. @@ -600,6 +613,200 @@ class PreparedTaggerComponent final { void* state_ = nullptr; }; +/// Prepared adapter for the deliberately narrow Reflux ABI. One callback receives four contiguous +/// faces of one rank-local child patch. It has no communicator, topology mask, global register or +/// live state; the enclosing PreparedAmrProgramRefluxTransition validates and publishes its result. +class PreparedRefluxComponent final { + public: + PreparedRefluxComponent(PreparedRefluxSpec spec, + std::shared_ptr component) + : spec_(std::move(spec)), component_(std::move(component)) { + validate_(); + prepare_provider_contract_(); + local_execution_ = std::make_shared( + spec_.execution->without_collective_authority()); + state_owner_ = component_->prepare_fresh_state( + POPS_NATIVE_INTERFACE_REFLUX_V1, spec_.interface_version, local_execution_->view()); + state_ = state_owner_.get(); + } + + [[nodiscard]] const std::string& provider_identity() const noexcept { + return spec_.provider_identity; + } + [[nodiscard]] std::string_view collective_contract() const noexcept { + return collective_contract_; + } + + void apply(const PreparedAmrRefluxLocalRequest& request) const { + static_assert(sizeof(Real) == sizeof(double), + "Reflux ABI v1 requires the binary64 PoPS backend"); + if (request.transition_identity == nullptr || request.transition_identity->empty() || + request.patch_identity == nullptr || request.patch_identity->empty() || + request.parent_level < 0 || request.child_level != request.parent_level + 1 || + request.logical_time.level != request.parent_level || request.logical_time.macro_step < 0 || + request.coarse.components <= 0 || request.coarse.components != request.fine.components || + request.coarse.components != request.correction.components || + request.coarse.I0 != request.fine.I0 || request.coarse.I1 != request.fine.I1 || + request.coarse.J0 != request.fine.J0 || request.coarse.J1 != request.fine.J1 || + request.coarse.I0 != request.correction.I0 || request.coarse.I1 != request.correction.I1 || + request.coarse.J0 != request.correction.J0 || request.coarse.J1 != request.correction.J1 || + !std::isfinite(request.dx) || !std::isfinite(request.dy) || request.dx <= Real(0) || + request.dy <= Real(0)) + throw std::invalid_argument("prepared native Reflux invocation is incomplete"); + for (const std::string* identity : request.interface_identities) + if (identity == nullptr || identity->empty()) + throw std::invalid_argument("prepared native Reflux face identity is empty"); + + const auto make_const_view = [&](const Real* data, int axis) { + if (data == nullptr) + throw std::invalid_argument("prepared native Reflux input face is absent"); + const std::size_t tangent = + static_cast(axis == 0 ? request.coarse.J1 - request.coarse.J0 + 1 + : request.coarse.I1 - request.coarse.I0 + 1); + const std::size_t components = static_cast(request.coarse.components); + const std::size_t extent0 = axis == 0 ? 1u : tangent; + const std::size_t extent1 = axis == 0 ? tangent : 1u; + return PopsConstFieldViewV1{sizeof(PopsConstFieldViewV1), + data, + 2, + {extent0, extent1, 1}, + {static_cast(extent1 * components), + static_cast(components), 0}, + components, + 1, + POPS_FIELD_CENTERING_FACE_V1, + 1u << static_cast(axis), + {0, 0, 0}, + {0, 0, 0}, + POPS_SCALAR_FLOAT64_V1, + POPS_MEMORY_SPACE_HOST_V1, + spec_.layout_identity.c_str(), + request.patch_identity->c_str(), + POPS_FIELD_OWNERSHIP_RUNTIME_BORROWED_V1}; + }; + const auto make_output_view = [&](Real* data, int axis) { + if (data == nullptr) + throw std::invalid_argument("prepared native Reflux output face is absent"); + const std::size_t tangent = + static_cast(axis == 0 ? request.coarse.J1 - request.coarse.J0 + 1 + : request.coarse.I1 - request.coarse.I0 + 1); + const std::size_t components = static_cast(request.coarse.components); + const std::size_t extent0 = axis == 0 ? 1u : tangent; + const std::size_t extent1 = axis == 0 ? tangent : 1u; + return PopsFieldViewV1{sizeof(PopsFieldViewV1), + data, + 2, + {extent0, extent1, 1}, + {static_cast(extent1 * components), + static_cast(components), 0}, + components, + 1, + POPS_FIELD_CENTERING_CELL_V1, + 0, + {0, 0, 0}, + {0, 0, 0}, + POPS_SCALAR_FLOAT64_V1, + POPS_MEMORY_SPACE_HOST_V1, + spec_.layout_identity.c_str(), + request.patch_identity->c_str(), + POPS_FIELD_OWNERSHIP_RUNTIME_BORROWED_V1}; + }; + + const std::array coarse{request.coarse.cL, request.coarse.cR, request.coarse.cB, + request.coarse.cT}; + const std::array fine{request.fine.fL, request.fine.fR, request.fine.fB, + request.fine.fT}; + const std::array correction{request.correction.x_low, request.correction.x_high, + request.correction.y_low, request.correction.y_high}; + const std::array axes{0, 0, 1, 1}; + const std::array sides{ + POPS_REFLUX_FACE_LOW_V1, POPS_REFLUX_FACE_HIGH_V1, POPS_REFLUX_FACE_LOW_V1, + POPS_REFLUX_FACE_HIGH_V1}; + std::array faces; + for (std::size_t face = 0; face < faces.size(); ++face) + faces[face] = PopsRefluxFaceV1{ + sizeof(PopsRefluxFaceV1), + request.interface_identities[face]->c_str(), + axes[face], + sides[face], + static_cast(Real(1) / (axes[face] == 0 ? request.dx : request.dy)), + make_const_view(coarse[face], axes[face]), + make_const_view(fine[face], axes[face]), + make_output_view(correction[face], axes[face])}; + + const PopsLogicalTimeV1 logical_time{sizeof(PopsLogicalTimeV1), + spec_.clock_identity.c_str(), + request.logical_time.macro_step, + request.parent_level, + 0, + 0, + request.logical_time.phase.numerator, + request.logical_time.phase.denominator, + 0.0, + request.logical_time.physical_time}; + const PopsRefluxRequestV1 abi_request{sizeof(PopsRefluxRequestV1), + request.transition_identity->c_str(), + request.parent_level, + request.child_level, + faces.size(), + faces.data(), + logical_time, + local_execution_->view()}; + PopsComponentStatusV1 status = component::unwritten_component_status(); + const auto& api = component_->table(POPS_NATIVE_INTERFACE_REFLUX_V1, + spec_.interface_version); + const int code = component::apply_reflux_interface_batch(api, state_, abi_request, status); + if (code != 0) + throw std::runtime_error(status.reason == nullptr ? "native Reflux component failed" + : status.reason); + } + + private: + void prepare_provider_contract_() { + ExactContractBuilder contract; + contract.text("pops.runtime.external-amr-reflux-provider") + .scalar(std::uint32_t{1}) + .text(spec_.provider_identity) + .text(spec_.component_id) + .text(spec_.manifest_identity) + .text(spec_.layout_identity) + .text(spec_.clock_identity) + .scalar(spec_.interface_version) + .text(spec_.execution->identity()); + collective_contract_ = std::move(contract).release(); + } + + void validate_() const { + if (!component_ || !spec_.execution || spec_.provider_identity.empty() || + spec_.component_id.empty() || spec_.manifest_identity.empty() || + spec_.layout_identity.empty() || spec_.clock_identity.empty() || + spec_.interface_version != 1) + throw std::invalid_argument("prepared AMR Reflux specification is incomplete"); + if constexpr (!std::is_same_v) + throw std::invalid_argument( + "prepared external Reflux v1 is qualified only for a host execution backend"); + component::validate_execution_context(spec_.execution->view()); + if (spec_.execution->view().memory_space != POPS_MEMORY_SPACE_HOST_V1) + throw std::invalid_argument( + "prepared external Reflux v1 requires host-resident face storage"); + const auto& api = component_->api(); + if (api.component_id == nullptr || api.manifest_identity == nullptr || + spec_.component_id != api.component_id || spec_.manifest_identity != api.manifest_identity) + throw std::invalid_argument("prepared AMR Reflux changed native component identity"); + component::require_operation( + component_->table(POPS_NATIVE_INTERFACE_REFLUX_V1, spec_.interface_version) + .apply_interface_batch != nullptr, + "apply_interface_batch"); + } + + PreparedRefluxSpec spec_; + std::shared_ptr component_; + std::shared_ptr local_execution_; + component::LoadedComponent::PreparedState state_owner_; + void* state_ = nullptr; + std::string collective_contract_; +}; + /// External Clustering ABI contract: each result is `2 * dimension` signed integers laid out as /// `[lo_0, ..., lo_(d-1), hi_0, ..., hi_(d-1)]`, inclusive and relative to the supplied region. class PreparedClusteringComponent final : public pops::amr::ClusteringProvider { diff --git a/include/pops/runtime/amr_system.hpp b/include/pops/runtime/amr_system.hpp index 7600c0c01..af778c882 100644 --- a/include/pops/runtime/amr_system.hpp +++ b/include/pops/runtime/amr_system.hpp @@ -384,6 +384,8 @@ class AmrSystem { POPS_EXPORT void install_amr_clustering_component( runtime::amr::PreparedClusteringSpec spec, std::shared_ptr component); + POPS_EXPORT void install_amr_reflux_component( + runtime::amr::PreparedRefluxSpec spec, std::shared_ptr component); POPS_EXPORT void discard_amr_provider_components(); /// Materialize one exact shared NumericalFlux route on a frozen AMR level. This seam is called /// only after the lazy AmrRuntime has been built and before bind freezes composition. diff --git a/python/bindings/core/init/init_amr.cpp b/python/bindings/core/init/init_amr.cpp index c9a3b55be..ff7b5e8e7 100644 --- a/python/bindings/core/init/init_amr.cpp +++ b/python/bindings/core/init/init_amr.cpp @@ -182,6 +182,19 @@ pops::runtime::amr::PreparedClusteringSpec amr_clustering_spec_from_python( return spec; } +pops::runtime::amr::PreparedRefluxSpec amr_reflux_spec_from_python(const py::dict& row, + const py::dict& execution) { + pops::runtime::amr::PreparedRefluxSpec spec; + spec.provider_identity = py::cast(row["provider_identity"]); + spec.component_id = py::cast(row["component_id"]); + spec.manifest_identity = py::cast(row["component_manifest_identity"]); + spec.layout_identity = py::cast(row["layout_identity"]); + spec.clock_identity = py::cast(row["clock_identity"]); + spec.interface_version = py::cast(row["interface_version"]); + spec.execution = pops::python::detail::make_component_execution_context(execution); + return spec; +} + // Assembly seams: per-block composition, native block, and refinement tagging. void bind_amr_assembly(py::class_& cls) { cls.def(py::init()) @@ -272,6 +285,14 @@ void bind_amr_assembly(py::class_& cls) { amr_clustering_spec_from_python(binding, execution), std::move(component)); }, py::arg("component"), py::arg("binding"), py::arg("execution_context")) + .def( + "_install_amr_reflux_component", + [](AmrSystem& system, std::shared_ptr component, + const py::dict& binding, const py::dict& execution) { + system.install_amr_reflux_component(amr_reflux_spec_from_python(binding, execution), + std::move(component)); + }, + py::arg("component"), py::arg("binding"), py::arg("execution_context")) .def("_discard_amr_provider_components", &AmrSystem::discard_amr_provider_components, "Roll back one failed external AMR provider transaction.") .def( diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index ae23d0cd3..1487f7ec7 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -246,6 +246,7 @@ struct AmrSystem::Impl { std::map field_storage_routes_; std::shared_ptr amr_tagger_component_; std::shared_ptr amr_clustering_component_; + std::shared_ptr amr_reflux_component_; struct BootstrapArray { std::string centering; int ncomp = 0; @@ -890,6 +891,9 @@ struct AmrSystem::Impl { runtime->install_external_tagger(amr_tagger_component_); if (amr_clustering_component_) runtime->install_external_clustering(amr_clustering_component_); + // Reflux selection is a collective optional-provider contract: every rank enters this call, + // including ranks where no external provider was selected. + runtime->install_external_reflux(amr_reflux_component_); if (!boundary_plans_.empty()) runtime->install_boundary_storage_routes(field_storage_routes_); // Low-level facade compatibility has no authored AMRTransfer object. Resolve its exact @@ -1490,6 +1494,17 @@ POPS_EXPORT void AmrSystem::install_amr_clustering_component( std::move(spec), std::move(component)); } +POPS_EXPORT void AmrSystem::install_amr_reflux_component( + runtime::amr::PreparedRefluxSpec spec, std::shared_ptr component) { + Impl* P = p_.get(); + require_assembling_amr(P->bound_, "install_amr_reflux_component"); + if (P->built || P->amr_reflux_component_) + throw std::runtime_error( + "AmrSystem external Reflux requires one installation before runtime build"); + P->amr_reflux_component_ = std::make_shared( + std::move(spec), std::move(component)); +} + POPS_EXPORT void AmrSystem::discard_amr_provider_components() { Impl* P = p_.get(); require_assembling_amr(P->bound_, "discard_amr_provider_components"); @@ -1497,6 +1512,7 @@ POPS_EXPORT void AmrSystem::discard_amr_provider_components() { throw std::runtime_error("AmrSystem cannot discard AMR providers after runtime build"); P->amr_tagger_component_.reset(); P->amr_clustering_component_.reset(); + P->amr_reflux_component_.reset(); } POPS_EXPORT void AmrSystem::install_interface_flux_component( From 1f2add44df29311d64edb317f31198a434ecaba9 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:34:16 +0200 Subject: [PATCH 060/109] test(architecture): fence prepared Reflux authority --- ...TION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md | 20 ++- ...prepared_reflux_runtime_execution_fence.py | 132 ++++++++++++++++++ 2 files changed, 145 insertions(+), 7 deletions(-) create mode 100644 tests/python/architecture/test_prepared_reflux_runtime_execution_fence.py diff --git a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md index d279bfee2..935aac84b 100644 --- a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md +++ b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md @@ -1418,13 +1418,19 @@ taille/header de table et opérations requises avant de conserver le handle de b sont résolues une fois à l'installation ; aucun `dlsym`, nom de classe ou dispatch Python n'entre dans une boucle de cellules. -Le contrat `Reflux` v1 est volontairement livré avant son branchement dans -`PreparedAmrProgramRefluxTransition` : catalogue, manifest, loader et consumer typé peuvent qualifier -un conformer, mais le runtime AMR continue d'utiliser son kernel interne tant qu'un adaptateur préparé -ne peut pas fournir les vues locales sans dupliquer le ledger ni transférer l'autorité collective. Une -configuration AMR ne prétend donc pas encore avoir sélectionné un provider `Reflux` externe. Cette -première qualification est limitée à la cible 2D, `float64`, CPU déjà admise par le loader de -composants ; elle ne constitue pas une promesse GPU. +Le contrat `Reflux` v1 possède maintenant un adaptateur préparé interne vers +`PreparedAmrProgramRefluxTransition`. Pour chaque patch enfant local, l'adaptateur reçoit quatre +paires de flux déjà intégrés et écrit quatre corrections dans des buffers persistants empoisonnés +avant l'appel. PoPS vérifie que chaque valeur a été écrite et reste finie, atteint un consensus +d'échec entre rangs, puis applique seul périodicité, masque de couverture, réduction MPI et +publication transactionnelle. La présence et le contrat exact du provider sont également comparés +entre rangs avant toute exécution. + +Cette tranche ne publie pas encore la sélection `Reflux` dans la résolution normalisée des providers +AMR : le seam d'installation demeure interne et les configurations publiques continuent donc +d'utiliser le kernel builtin. La qualification initiale de l'adaptateur reste limitée à la cible 2D, +`float64`, CPU avec stockage hôte. Le chemin n'est pas encore prouvé par compilation native, exécution +MPI avec un composant externe, mesure de conservation ni backend GPU. Les champs sémantiques inconnus, capacités sans preuve, collisions d'identité et entry points manquants sont refusés. Un vieux manifest n'est pas « réparé » silencieusement. diff --git a/tests/python/architecture/test_prepared_reflux_runtime_execution_fence.py b/tests/python/architecture/test_prepared_reflux_runtime_execution_fence.py new file mode 100644 index 000000000..9616dbb72 --- /dev/null +++ b/tests/python/architecture/test_prepared_reflux_runtime_execution_fence.py @@ -0,0 +1,132 @@ +"""ADC-681: a prepared Reflux component executes without owning AMR authority.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +PATCH_RANGE = ( + ROOT / "include" / "pops" / "numerics" / "time" / "amr" / "levels" + / "amr_patch_range.hpp" +) +SUBCYCLING = ( + ROOT / "include" / "pops" / "numerics" / "time" / "amr" / "levels" + / "amr_subcycling.hpp" +) +PROVIDERS = ( + ROOT / "include" / "pops" / "runtime" / "amr" + / "prepared_component_providers.hpp" +) +AMR_RUNTIME = ROOT / "include" / "pops" / "runtime" / "amr" / "amr_runtime.hpp" +PROGRAM_REFLUX = ( + ROOT / "include" / "pops" / "runtime" / "amr" / "amr_program_reflux.hpp" +) +PROGRAM_CONTEXT = ( + ROOT / "include" / "pops" / "runtime" / "program" / "amr_program_context.hpp" +) +AMR_SYSTEM = ROOT / "src" / "runtime" / "amr" / "amr_system.cpp" +AMR_BINDING = ROOT / "python" / "bindings" / "core" / "init" / "init_amr.cpp" +RUNTIME_AUTHORITIES = ROOT / "python" / "pops" / "runtime" / "_runtime_authorities.py" + + +def _between(text: str, begin: str, end: str) -> str: + return text.split(begin, 1)[1].split(end, 1)[0] + + +def test_transition_executes_local_kernel_before_pops_collective_publication() -> None: + source = SUBCYCLING.read_text() + transition = _between( + source, + "class PreparedAmrProgramRefluxTransition", + "class PreparedAmrProgramRefluxPlan", + ) + assert "PreparedAmrRefluxLocalKernel local_kernel_" in transition + assert "workspace.poison();" in transition + assert "local_kernel_(PreparedAmrRefluxLocalRequest{" in transition + assert "workspace.all_finite()" in transition + assert "all_reduce_or_inplace(&preflight_consensus" in transition + assert "prepared Reflux provider differs between communicator ranks" in transition + assert transition.index("local_kernel_(PreparedAmrRefluxLocalRequest{") < ( + transition.index("route_prepared_reflux_correction_") + ) + assert transition.index("all_reduce_max(local_failure") < transition.index( + "correction_.gather(communicator);" + ) + assert "route_reflux_integrated_pair_prevalidated_" in transition + assert "apply_reflux_interface_batch" not in transition + + +def test_component_adapter_is_host_local_noncollective_and_has_no_topology() -> None: + source = PROVIDERS.read_text() + adapter = _between( + source, + "class PreparedRefluxComponent final", + "/// External Clustering ABI contract", + ) + assert "without_collective_authority()" in adapter + assert "collective_contract() const noexcept" in adapter + assert "POPS_MEMORY_SPACE_HOST_V1" in adapter + assert "apply_reflux_interface_batch" in adapter + assert "POPS_NATIVE_INTERFACE_REFLUX_V1" in adapter + assert "FluxRegister" not in adapter + assert "CoverageMask" not in adapter + assert "all_reduce" not in adapter + + +def test_pops_maps_validated_faces_through_coverage_and_periodicity() -> None: + source = PATCH_RANGE.read_text() + kernel = _between( + source, + "struct RoutePreparedRefluxCorrectionKernel", + "} // namespace detail", + ) + assert "canonicalize" in kernel + assert "coverage.covered" in kernel + assert "correction.add" in kernel + assert "faces.x_low[index]" in kernel + assert "faces.x_high[index]" in kernel + assert "faces.y_low[index]" in kernel + assert "faces.y_high[index]" in kernel + + +def test_runtime_installation_reprepares_transitions_and_routes_logical_time() -> None: + runtime = AMR_RUNTIME.read_text() + install = _between( + runtime, + "void install_external_reflux(", + "/// Inject the current Program evaluation coordinate", + ) + assert "external_reflux_ = std::move(provider);" in install + assert "require_prepared_provider_collective_consensus" in install + assert "rematerialize_persistent_topology_resources_" in install + rematerialize = _between( + runtime, + "void rematerialize_persistent_topology_resources_(", + "void record_topology_replacement_()", + ) + assert "provider->apply(request);" in rematerialize + assert "external_reflux_kernel, block.state_identity" in rematerialize + + route = PROGRAM_REFLUX.read_text() + assert "const amr::ClockStamp& logical_time" in route + assert "&logical_time" in route + context = PROGRAM_CONTEXT.read_text() + assert ( + "route_reflux_program(*eng_, sb, child, coarse_role, fine_role, sync_clock)" + in context + ) + + +def test_internal_install_seam_exists_without_claiming_public_resolution() -> None: + system = AMR_SYSTEM.read_text() + binding = AMR_BINDING.read_text() + authorities = RUNTIME_AUTHORITIES.read_text() + assert "install_amr_reflux_component(" in system + assert "runtime->install_external_reflux(amr_reflux_component_);" in system + assert "if (amr_reflux_component_)" not in _between( + system, + "runtime->install_external_tagger(amr_tagger_component_);", + "if (!boundary_plans_.empty())", + ) + assert '"_install_amr_reflux_component"' in binding + assert '"_install_amr_reflux_component"' not in authorities + assert 'tuple(providers) != ("clustering", "tagger")' in authorities From df0449582dd874b68d5b06f87185d2212145343b Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:51:43 +0200 Subject: [PATCH 061/109] fix(amr): include logical clock in reflux contract --- include/pops/numerics/time/amr/levels/amr_subcycling.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/include/pops/numerics/time/amr/levels/amr_subcycling.hpp b/include/pops/numerics/time/amr/levels/amr_subcycling.hpp index ac48aaece..fc16d48b2 100644 --- a/include/pops/numerics/time/amr/levels/amr_subcycling.hpp +++ b/include/pops/numerics/time/amr/levels/amr_subcycling.hpp @@ -3,6 +3,7 @@ #include #include // coarsen, parallel_copy #include +#include #include #include From 61bd2f3a2b85cef590219c348e230a4f6628f0c3 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:52:42 +0200 Subject: [PATCH 062/109] fix(codegen): bind qualified flux provider packs (ADC-682) --- include/pops/numerics/fv/flux_interfaces.hpp | 1 + python/pops/codegen/_compiler_lowering.py | 9 ++ .../pops/codegen/component_provider_packs.py | 104 ++++++++++++++++++ python/pops/codegen/module_emit_brick.py | 6 +- python/pops/codegen/module_lowering.py | 35 ++---- python/pops/model/provider_pack.py | 55 ++++++++- python/pops/physics/_authoring_view.py | 73 ++++++++++-- python/pops/physics/_facade_compile.py | 16 +++ 8 files changed, 265 insertions(+), 34 deletions(-) create mode 100644 python/pops/codegen/component_provider_packs.py diff --git a/include/pops/numerics/fv/flux_interfaces.hpp b/include/pops/numerics/fv/flux_interfaces.hpp index 0265fad9f..a8e33b31d 100644 --- a/include/pops/numerics/fv/flux_interfaces.hpp +++ b/include/pops/numerics/fv/flux_interfaces.hpp @@ -96,6 +96,7 @@ struct QualifiedProviderRequirement { const char* layout; const char* value_kind; const char* producer; + bool available; int storage_slot; }; diff --git a/python/pops/codegen/_compiler_lowering.py b/python/pops/codegen/_compiler_lowering.py index 9d45e150e..885ec68aa 100644 --- a/python/pops/codegen/_compiler_lowering.py +++ b/python/pops/codegen/_compiler_lowering.py @@ -12,6 +12,7 @@ class _CompilerEmitter(Protocol): """Minimal executable half of a compiler lowering.""" def check(self) -> object: ... + def __pops_bind_component_provider_packs__(self, packs: Any) -> None: ... def __pops_native_loader_source__( self, *, name: Any = None, target: str = "system", hoist_reciprocals: bool = False, @@ -26,6 +27,14 @@ class CompilerLowering: source_module: Module facade: object + def bind_component_provider_packs(self, packs: Any) -> None: + """Bind one resolved provider-pack authority before native source emission.""" + result = self.emit_model.__pops_bind_component_provider_packs__(packs) + if result is not None: + raise TypeError( + "compiler provider-pack binding protocol must return None" + ) + def native_loader_source( self, *, name: Any = None, target: str = "system", hoist_reciprocals: bool = False, diff --git a/python/pops/codegen/component_provider_packs.py b/python/pops/codegen/component_provider_packs.py new file mode 100644 index 000000000..0b9664895 --- /dev/null +++ b/python/pops/codegen/component_provider_packs.py @@ -0,0 +1,104 @@ +"""Exact component-provider packs shared by every compiler entry route. + +The operator-first :class:`pops.model.Module` is the authority for provider identity. Kernel +emitters must not rediscover providers from the legacy auxiliary layout: this module resolves the +full pack, every per-operator subset, and the physical-flux subset once and passes that immutable +value through the explicit compiler-emitter protocol. +""" +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Any + +from pops.model.provider_pack import ( + ProviderPack, + build_operator_provider_pack, + build_provider_pack, +) + + +@dataclass(frozen=True, slots=True) +class ComponentProviderPacks: + """One immutable provider resolution for a canonical Module.""" + + complete: ProviderPack + by_operator: Mapping[str, ProviderPack] + physical_flux: ProviderPack + + def __post_init__(self) -> None: + if type(self.complete) is not ProviderPack: + raise TypeError("ComponentProviderPacks.complete must be an exact ProviderPack") + rows = dict(self.by_operator) + if any(not isinstance(name, str) or not name for name in rows): + raise TypeError( + "ComponentProviderPacks.by_operator keys must be non-empty strings" + ) + if any(type(pack) is not ProviderPack for pack in rows.values()): + raise TypeError( + "ComponentProviderPacks.by_operator values must be exact ProviderPack values" + ) + object.__setattr__(self, "by_operator", MappingProxyType(rows)) + if type(self.physical_flux) is not ProviderPack: + raise TypeError( + "ComponentProviderPacks.physical_flux must be an exact ProviderPack" + ) + + def attach(self, target: Any) -> None: + """Attach compiler-owned immutable evidence to one emitter carrier. + + Reattachment is idempotent and verifies byte-for-byte logical equality. This is needed + because a facade and its private formula carrier are distinct Python objects but emit one + native package; neither may retain a different provider resolution. + """ + values = { + "_component_provider_pack": self.complete, + "_component_provider_metadata": self.complete.to_data(), + "_component_operator_provider_packs": self.by_operator, + "_component_operator_provider_metadata": MappingProxyType({ + name: pack.to_data() for name, pack in self.by_operator.items() + }), + "_component_flux_provider_pack": self.physical_flux, + "_component_flux_provider_metadata": self.physical_flux.to_data(), + } + + def canonical(value: Any) -> Any: + if isinstance(value, ProviderPack): + return value.to_data() + if isinstance(value, Mapping): + return { + key: canonical(item) + for key, item in value.items() + } + return value + + for name, value in values.items(): + previous = getattr(target, name, None) + if previous is not None and canonical(previous) != canonical(value): + raise ValueError( + "compiler emitter retained a conflicting component-provider pack" + ) + object.__setattr__(target, name, value) + + +def resolve_component_provider_packs(module: Any) -> ComponentProviderPacks: + """Resolve all exact provider packs from one canonical Module authority.""" + complete = build_provider_pack(module) + by_operator = { + operator.name: build_operator_provider_pack(module, operator) + for operator in module.operator_registry() + } + flux_requirements = [] + for operator in module.operator_registry(): + if operator.kind == "grid_operator": + flux_requirements.extend(by_operator[operator.name]) + physical_flux = complete.select(flux_requirements) + return ComponentProviderPacks( + complete=complete, + by_operator=by_operator, + physical_flux=physical_flux, + ) + + +__all__ = ["ComponentProviderPacks", "resolve_component_provider_packs"] diff --git a/python/pops/codegen/module_emit_brick.py b/python/pops/codegen/module_emit_brick.py index 1bdc4c745..7389c5e76 100644 --- a/python/pops/codegen/module_emit_brick.py +++ b/python/pops/codegen/module_emit_brick.py @@ -241,8 +241,10 @@ def roles_init(roles: Any) -> Any: contract["representation"], contract["centering"], contract["unit"] or "", contract["layout"], contract["value_kind"] or "", provider["producer"] or "", ] - S.append(" {%s, %d}," % - (", ".join(json.dumps(value) for value in values), provider["slot"])) + availability = "true" if provider["availability"] else "false" + S.append(" {%s, %s, %d}," % + (", ".join(json.dumps(value) for value in values), + availability, provider["slot"])) S.append(" }};") if rt_member: # member pops::RuntimeParams params{count, {defaults}} (P7-b) S.append(rt_member.rstrip("\n")) diff --git a/python/pops/codegen/module_lowering.py b/python/pops/codegen/module_lowering.py index 56cce3234..fd1f1fb34 100644 --- a/python/pops/codegen/module_lowering.py +++ b/python/pops/codegen/module_lowering.py @@ -20,7 +20,6 @@ from __future__ import annotations -from types import MappingProxyType from collections.abc import Iterable, Mapping from typing import Any, cast @@ -75,30 +74,13 @@ def _body_for_state(body: Any) -> Any: # Preserve the canonical source-Module identity across the internal facade lowering. The # resulting CompiledModel authenticates this scalar hash; it never retains ``module`` itself. object.__setattr__(m, "_compile_source_module_hash", module.module_hash()) - from pops.model.provider_pack import ( # noqa: PLC0415 - build_operator_provider_pack, - build_provider_pack, + from pops.codegen.component_provider_packs import ( # noqa: PLC0415 + resolve_component_provider_packs, ) - provider_pack = build_provider_pack(module) - object.__setattr__(m, "_component_provider_pack", provider_pack) - object.__setattr__(m, "_component_provider_metadata", provider_pack.to_data()) - operator_provider_packs = { - operator.name: build_operator_provider_pack(module, operator) - for operator in module.operator_registry() - } - object.__setattr__(m, "_component_operator_provider_packs", - MappingProxyType(operator_provider_packs)) - object.__setattr__(m, "_component_operator_provider_metadata", MappingProxyType({ - name: pack.to_data() for name, pack in operator_provider_packs.items() - })) - flux_keys = [] - for operator in module.operator_registry(): - if operator.kind == "grid_operator": - flux_keys.extend(operator_provider_packs[operator.name]) - flux_provider_pack = provider_pack.select(flux_keys) - object.__setattr__(m, "_component_flux_provider_pack", flux_provider_pack) - object.__setattr__(m, "_component_flux_provider_metadata", flux_provider_pack.to_data()) + m.__pops_bind_component_provider_packs__( + resolve_component_provider_packs(module) + ) # The facade is a lowering view of THIS Module, not a newly declared model. Re-anchor its empty # backing model before the first declaration so every derived operator registry retains the # Module's exact authoring authority. Without this, owner-qualified Program nodes would be @@ -191,7 +173,7 @@ def _declare_aux(nm: Any, key: Any) -> None: coverage_rows.append(LoweringCoverageRow( "module:%s:eigenvalues" % module.name, "documentary")) - for key in provider_pack: + for key in m._component_provider_pack: key_data = key.to_data() stable_key = "%s/%s/%s" % ( key_data["space_kind"], key_data["space_name"], key_data["component"]) @@ -441,6 +423,11 @@ def lower_and_validate(model: Any, facade: Any = None, state_space: Any = None) lowering = require_compiler_lowering(model) if diagnostic_facade is None: diagnostic_facade = lowering.facade + from pops.codegen.component_provider_packs import resolve_component_provider_packs + + lowering.bind_component_provider_packs( + resolve_component_provider_packs(lowering.source_module) + ) states = lowering.source_module.state_spaces() if len(states) > 1: emit_model = _module_to_model( diff --git a/python/pops/model/provider_pack.py b/python/pops/model/provider_pack.py index 16d42b251..fa2a8ba50 100644 --- a/python/pops/model/provider_pack.py +++ b/python/pops/model/provider_pack.py @@ -256,6 +256,50 @@ def select_spaces(self, *, owner_qid: str, (owner_qid, sorted(missing))) return self.select(keys) + def select_components( + self, + *, + owner_qid: str, + spaces: Iterable[tuple[str, str]], + components: Iterable[str], + ) -> ProviderPack: + """Select exact components from declared spaces without a bare-name fallback. + + Component spelling is only a filter inside the already-qualified owner/space set. A + missing component or the same spelling in two selected spaces is rejected rather than + guessed, so an operator that needs one of two homonymous fields must qualify its input + space more narrowly. + """ + _non_empty(owner_qid, "ProviderPack selection owner_qid") + requested_spaces = set(spaces) + requested_components = tuple(components) + if any(not isinstance(name, str) or not name for name in requested_components): + raise TypeError( + "ProviderPack components must contain non-empty strings" + ) + if len(set(requested_components)) != len(requested_components): + raise ValueError("ProviderPack components contains a duplicate") + candidates = [ + key for key in self + if key.owner_qid == owner_qid + and (key.space_kind, key.space_name) in requested_spaces + ] + selected = [] + for component in requested_components: + matches = [key for key in candidates if key.component == component] + if not matches: + raise MissingInputProvider( + "missing component %r in qualified provider spaces %r for owner %r" + % (component, sorted(requested_spaces), owner_qid) + ) + if len(matches) != 1: + raise MissingInputProvider( + "ambiguous component %r in qualified provider spaces %r for owner %r" + % (component, sorted(requested_spaces), owner_qid) + ) + selected.append(matches[0]) + return self.select(selected) + def to_data(self) -> dict[str, Any]: rows = [] for key in sorted(self._entries): @@ -363,7 +407,16 @@ def build_operator_provider_pack(module: Any, operator: Any) -> ProviderPack: spaces.append(("field", input_space.name)) if not spaces: return ProviderPack(capacity=full.capacity) - return full.select_spaces(owner_qid=str(module.owner_path.canonical()), spaces=spaces) + owner_qid = str(module.owner_path.canonical()) + requirements = getattr(operator, "requirements", {}) + required_components = requirements.get("aux", ()) + if required_components: + return full.select_components( + owner_qid=owner_qid, + spaces=spaces, + components=required_components, + ) + return full.select_spaces(owner_qid=owner_qid, spaces=spaces) __all__ = ["ComponentKey", "ComponentContract", "ProviderEntry", "ProviderPack", diff --git a/python/pops/physics/_authoring_view.py b/python/pops/physics/_authoring_view.py index 173ce2ae3..9a4a267b9 100644 --- a/python/pops/physics/_authoring_view.py +++ b/python/pops/physics/_authoring_view.py @@ -30,7 +30,23 @@ def _aux_name_set(self) -> Any: def _aux_requirements(self, exprs: Any) -> Any: """{'aux': [...]} of the aux fields the expressions read, or {} if none.""" aux_set = self._aux_name_set() - read = sorted(_dependencies(exprs) & aux_set) + dependencies = _dependencies(exprs) + pending = [name for name in dependencies if name in self.prim_defs] + expanded = set(dependencies) + visited = set() + while pending: + name = pending.pop() + if name in visited: + continue + visited.add(name) + nested = _dependencies((self.prim_defs[name],)) + expanded.update(nested) + pending.extend( + dependency + for dependency in nested + if dependency in self.prim_defs and dependency not in visited + ) + read = sorted(expanded & aux_set) return {"aux": read} if read else {} def state_space(self, name: str = "U") -> Any: @@ -85,25 +101,68 @@ def operator_registry(self, state_name: str = "U") -> Any: reg = _model.OperatorRegistry(owner=self.owner_path) state = self.state_space(state_name) fields = self.field_space() - aux_set = self._aux_name_set() def reads_fields(exprs: Any) -> bool: - return bool(_dependencies(exprs) & aux_set) + return bool(self._aux_requirements(exprs)) + + stability_exprs = [ + *self._eig.get("x", ()), + *self._eig.get("y", ()), + ] + if self._wave_speeds is not None: + stability_exprs.extend(self._wave_speeds["x"]) + stability_exprs.extend(self._wave_speeds["y"]) + if self._ws_jacobian is not None and self._ws_jacobian["rows"] is not None: + for direction in ("x", "y"): + stability_exprs.extend( + expression + for row in self._ws_jacobian["rows"][direction] + for expression in row + ) + if self._roe_rows is not None: + stability_exprs.extend(self._roe_rows["x"]) + stability_exprs.extend(self._roe_rows["y"]) + if self._roe_jacobian is not None: + for direction in ("x", "y"): + stability_exprs.extend( + expression + for row in self._roe_jacobian[direction] + for expression in row + ) # Flux divergence (grid_operator: State -> Rate(State)). if self._flux: + exprs = [ + *self._flux.get("x", ()), + *self._flux.get("y", ()), + *stability_exprs, + ] + rf = reads_fields(exprs) reg.register(_model.Operator( "flux_default", "grid_operator", - _model.Signature([state], _model.Rate(state)), + _model.Signature([state, fields] if rf else [state], + _model.Rate(state)), capabilities={"local": False, "linear": False, "produces_rate": True, "requires_ghosts": 1, "supports_device": True, - "default": True}, + "requires_fields": rf, "default": True}, + requirements=self._aux_requirements(exprs), source=None)) for nm in sorted(self._flux_terms): + term = self._flux_terms[nm] + exprs = [ + *term.get("x", ()), + *term.get("y", ()), + *stability_exprs, + ] + rf = reads_fields(exprs) reg.register(_model.Operator( - nm, "grid_operator", _model.Signature([state], _model.Rate(state)), + nm, "grid_operator", + _model.Signature([state, fields] if rf else [state], + _model.Rate(state)), capabilities={"local": False, "linear": False, "produces_rate": True, - "requires_ghosts": 1, "supports_device": True}, + "requires_ghosts": 1, "supports_device": True, + "requires_fields": rf}, + requirements=self._aux_requirements(exprs), source=None)) # Local sources (local_source: State[, Fields] -> Rate(State)). diff --git a/python/pops/physics/_facade_compile.py b/python/pops/physics/_facade_compile.py index 746de5ad1..f1ef03f44 100644 --- a/python/pops/physics/_facade_compile.py +++ b/python/pops/physics/_facade_compile.py @@ -40,11 +40,27 @@ def __pops_compiler_lowering__(self) -> Any: facade=self, ) + def __pops_bind_component_provider_packs__(self, packs: Any) -> None: + """Bind the exact Module provider resolution to both native-emitter carriers.""" + from pops.codegen.component_provider_packs import ComponentProviderPacks + + if type(packs) is not ComponentProviderPacks: + raise TypeError( + "compiler provider-pack binding requires exact ComponentProviderPacks" + ) + packs.attach(self) + packs.attach(self._m) + def __pops_native_loader_source__( self, *, name: Any = None, target: str = "system", hoist_reciprocals: bool = False, ) -> str: """Emit a native package without exposing the private formula carrier.""" + from pops.codegen.component_provider_packs import resolve_component_provider_packs + + self.__pops_bind_component_provider_packs__( + resolve_component_provider_packs(self.module) + ) return self._m.emit_cpp_native_loader( name=name, target=target, hoist_reciprocals=hoist_reciprocals) From d7b34014847ab7d1c71dff35b43cdd18024a1dab Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 06:52:47 +0200 Subject: [PATCH 063/109] test(codegen): prove exact flux provider identity (ADC-682) --- .../codegen/test_compiled_model_boundary.py | 3 + .../codegen/test_compiler_model_provider.py | 59 +++++++++++++++++++ .../codegen/test_component_provider_pack.py | 38 ++++++++++++ 3 files changed, 100 insertions(+) diff --git a/tests/python/unit/codegen/test_compiled_model_boundary.py b/tests/python/unit/codegen/test_compiled_model_boundary.py index 18c24cbe6..c58666ce4 100644 --- a/tests/python/unit/codegen/test_compiled_model_boundary.py +++ b/tests/python/unit/codegen/test_compiled_model_boundary.py @@ -32,6 +32,9 @@ def _model_hash(self): def check(self): return None + def __pops_bind_component_provider_packs__(self, packs): + self.provider_packs = packs + def __pops_native_loader_source__( self, *, name=None, target="system", hoist_reciprocals=False): return "// compiled-model-boundary fixture\n" diff --git a/tests/python/unit/codegen/test_compiler_model_provider.py b/tests/python/unit/codegen/test_compiler_model_provider.py index 7d0079d03..28f6e0aef 100644 --- a/tests/python/unit/codegen/test_compiler_model_provider.py +++ b/tests/python/unit/codegen/test_compiler_model_provider.py @@ -10,6 +10,7 @@ from pops.codegen.module_lowering import lower_and_validate from pops._ir.expr import Const from pops.model import Module, Rate +from pops.model.provider_pack import MissingInputProvider from pops.physics._facade import Model @@ -27,6 +28,21 @@ def _facade_model(name: str = "provider") -> Model: return model +def _field_dependent_flux_model(name: str, *, with_provider: bool = True) -> Model: + model = Model(name) + (rho,) = model.conservative_vars("rho") + phi = model.aux("phi") + grad_x = model.aux("grad_x") + model.aux("grad_y") + model.primitive_vars(rho=rho) + model.conservative_from([rho]) + model.flux(x=[rho * grad_x], y=[rho * grad_x]) + model.eigenvalues(x=(Const(1.0),), y=(Const(1.0),)) + if with_provider: + model.elliptic_rhs(rho + Const(0.0) * phi) + return model + + class _ThirdPartyProvider: """An external provider delegates only the documented compiler contract.""" @@ -50,6 +66,9 @@ class _CheckEmitter: def check(self) -> None: return None + def __pops_bind_component_provider_packs__(self, packs) -> None: + self.provider_packs = packs + def __pops_native_loader_source__( self, *, name=None, target="system", hoist_reciprocals=False): return "// test native loader\n" @@ -97,6 +116,46 @@ def test_frozen_module_remains_the_canonical_compiler_ir(): assert lowering.source_module is module +def test_facade_and_formula_carrier_share_one_minimal_flux_provider_pack(): + model = _field_dependent_flux_model("facade-flux-pack") + + emitted, source_module = lower_and_validate(model, facade=model) + + assert emitted is model + rows = model._component_flux_provider_metadata["entries"] + assert rows == model._m._component_flux_provider_metadata["entries"] + assert [row["key"]["component"] for row in rows] == ["grad_x"] + assert rows[0]["provider"]["availability"] is True + assert rows[0]["key"]["owner_qid"] == str(source_module.owner_path.canonical()) + + source = model.__pops_native_loader_source__() + assert rows[0]["key"]["owner_qid"] in source + assert '"grad_x"' in source + assert "true, 1" in source + + +def test_field_dependent_flux_without_provider_fails_before_native_source(): + model = _field_dependent_flux_model("missing-flux-provider", with_provider=False) + + with pytest.raises(MissingInputProvider, match="unset"): + lower_and_validate(model, facade=model) + + +def test_same_field_spelling_under_distinct_model_owners_stays_distinct_in_emitted_pack(): + left = _field_dependent_flux_model("left-flux-owner") + right = _field_dependent_flux_model("right-flux-owner") + + lower_and_validate(left, facade=left) + lower_and_validate(right, facade=right) + left_owner = left._component_flux_provider_metadata["entries"][0]["key"]["owner_qid"] + right_owner = right._component_flux_provider_metadata["entries"][0]["key"]["owner_qid"] + + assert left_owner != right_owner + assert left_owner in left.__pops_native_loader_source__() + assert right_owner not in left.__pops_native_loader_source__() + assert right_owner in right.__pops_native_loader_source__() + + class _MissingProtocol: pass diff --git a/tests/python/unit/codegen/test_component_provider_pack.py b/tests/python/unit/codegen/test_component_provider_pack.py index 170d0ca98..6a4b12f76 100644 --- a/tests/python/unit/codegen/test_component_provider_pack.py +++ b/tests/python/unit/codegen/test_component_provider_pack.py @@ -64,6 +64,29 @@ def test_minimal_selection_preserves_qualified_identity_and_refuses_missing_prov pack.select([ComponentKey("case/missing", "field", "electric", "grad_x")]) +def test_component_selection_is_space_qualified_and_refuses_homonym_ambiguity(): + contract = ComponentContract("field", "cell", "V/m", "cell") + left = ComponentKey("owner", "field", "left", "grad_x") + right = ComponentKey("owner", "field", "right", "grad_x") + pack = ProviderPack([ + (left, contract, ProviderEntry("left_solver", True, 0)), + (right, contract, ProviderEntry("right_solver", True, 0)), + ]) + + selected = pack.select_components( + owner_qid="owner", + spaces=(("field", "left"),), + components=("grad_x",), + ) + assert tuple(selected) == (left,) + with pytest.raises(MissingInputProvider, match="ambiguous component"): + pack.select_components( + owner_qid="owner", + spaces=(("field", "left"), ("field", "right")), + components=("grad_x",), + ) + + def test_operator_provider_pack_contains_fields_but_not_explicit_state_trace(): module = Module("operator_pack") state = module.state_space("U", ("rho",)) @@ -78,6 +101,21 @@ def test_operator_provider_pack_contains_fields_but_not_explicit_state_trace(): } +def test_operator_requirements_select_only_declared_components(): + module = Module("operator_component_pack") + state = module.state_space("U", ("rho",)) + fields = module.field_space("electric", ("phi", "grad_x", "grad_y")) + module.operator("solve", state >> fields, "field_operator", expr=1.0) + operator = SimpleNamespace( + signature=SimpleNamespace(inputs=(state, fields)), + requirements={"aux": ("grad_x",)}, + ) + + pack = build_operator_provider_pack(module, operator) + + assert tuple(key.component for key in pack) == ("grad_x",) + + def test_provider_pack_accepts_exact_capacity_and_refuses_capacity_plus_one_atomically(): first = _row("rho", 0) second = _row("mx", 1) From e5ef1033e8568005fb6c3da4e9599999f4ea7ad0 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:02:26 +0200 Subject: [PATCH 064/109] test(amr): execute prepared Reflux kernel --- .../amr/test_program_reflux_ledger.cpp | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/tests/cpp/integration/amr/test_program_reflux_ledger.cpp b/tests/cpp/integration/amr/test_program_reflux_ledger.cpp index b09dc2b0a..4791ff1a6 100644 --- a/tests/cpp/integration/amr/test_program_reflux_ledger.cpp +++ b/tests/cpp/integration/amr/test_program_reflux_ledger.cpp @@ -115,6 +115,110 @@ TEST(test_program_reflux_ledger, prepared_transition_does_not_skip_right_or_top_ << "every rejected preflight leaves the parent untouched"; } +TEST(test_program_reflux_ledger, prepared_local_kernel_routes_exact_face_corrections) { + const Box2D coarse_domain{{0, 0}, {7, 7}}; + const BoxArray coarse_boxes(std::vector{coarse_domain}); + const BoxArray fine_boxes(std::vector{Box2D{{4, 4}, {11, 11}}}); + const DistributionMapping coarse_mapping(coarse_boxes.size(), n_ranks()); + const DistributionMapping fine_mapping(fine_boxes.size(), n_ranks()); + AmrLevelMP coarse{MultiFab(coarse_boxes, coarse_mapping, 1, 0), nullptr, Real(0.5), Real(0.5)}; + AmrLevelMP fine{MultiFab(fine_boxes, fine_mapping, 1, 0), nullptr, Real(0.25), Real(0.25)}; + coarse.U.set_val(Real(0)); + fine.U.set_val(Real(0)); + + int calls = 0; + const auto kernel = [&calls](const PreparedAmrRefluxLocalRequest& request) { + ++calls; + EXPECT_EQ(*request.transition_identity, "pops://test/reflux"); + EXPECT_EQ(*request.patch_identity, "pops://test/reflux/patch=0"); + EXPECT_EQ(request.parent_level, 0); + EXPECT_EQ(request.child_level, 1); + EXPECT_EQ(request.global_child, 0u); + EXPECT_EQ(request.logical_time, (amr::ClockStamp{0, 3, amr::Rational(0, 1), 0.3})); + const auto x_size = + static_cast(request.correction.J1 - request.correction.J0 + 1) * + static_cast(request.correction.components); + const auto y_size = + static_cast(request.correction.I1 - request.correction.I0 + 1) * + static_cast(request.correction.components); + std::fill_n(request.correction.x_low, x_size, Real(3)); + std::fill_n(request.correction.x_high, x_size, Real(4)); + std::fill_n(request.correction.y_low, y_size, Real(5)); + std::fill_n(request.correction.y_high, y_size, Real(6)); + }; + auto transition = PreparedAmrProgramRefluxTransition::prepare_with_local_kernel( + coarse, fine, coarse_domain, Periodicity{false, false}, 0, "pops://test/reflux", kernel, + world_communicator_view()); + + EdgeStrip coarse_role = make_strip(2, 5, 2, 5, 1); + EdgeStrip fine_role = make_strip(2, 5, 2, 5, 1); + coarse_role.cL.assign(4, Real(1)); + coarse_role.cR.assign(4, Real(1)); + coarse_role.cB.assign(4, Real(1)); + coarse_role.cT.assign(4, Real(1)); + fine_role.fL.assign(4, Real(2)); + fine_role.fR.assign(4, Real(2)); + fine_role.fB.assign(4, Real(2)); + fine_role.fT.assign(4, Real(2)); + const amr::ClockStamp logical_time{0, 3, amr::Rational(0, 1), 0.3}; + transition.synchronize_integrated( + coarse.U, coarse.dx, coarse.dy, std::vector{coarse_role}, + std::vector{fine_role}, world_communicator_view(), &logical_time); + + EXPECT_EQ(calls, 1); + ASSERT_EQ(coarse.U.local_size(), 1); + EXPECT_EQ(coarse.U.fab(0)(1, 3, 0), Real(3)); + EXPECT_EQ(coarse.U.fab(0)(6, 3, 0), Real(4)); + EXPECT_EQ(coarse.U.fab(0)(3, 1, 0), Real(5)); + EXPECT_EQ(coarse.U.fab(0)(3, 6, 0), Real(6)); + EXPECT_EQ(coarse.U.fab(0)(2, 2, 0), Real(0)) + << "covered parent cells remain outside the sparse correction"; +} + +TEST(test_program_reflux_ledger, prepared_local_kernel_rejects_unwritten_output_atomically) { + const Box2D coarse_domain{{0, 0}, {7, 7}}; + const BoxArray coarse_boxes(std::vector{coarse_domain}); + const BoxArray fine_boxes(std::vector{Box2D{{4, 4}, {11, 11}}}); + const DistributionMapping coarse_mapping(coarse_boxes.size(), n_ranks()); + const DistributionMapping fine_mapping(fine_boxes.size(), n_ranks()); + AmrLevelMP coarse{MultiFab(coarse_boxes, coarse_mapping, 1, 0), nullptr, Real(0.5), Real(0.5)}; + AmrLevelMP fine{MultiFab(fine_boxes, fine_mapping, 1, 0), nullptr, Real(0.25), Real(0.25)}; + coarse.U.set_val(Real(0)); + fine.U.set_val(Real(0)); + + const auto incomplete = [](const PreparedAmrRefluxLocalRequest& request) { + const auto x_size = + static_cast(request.correction.J1 - request.correction.J0 + 1) * + static_cast(request.correction.components); + std::fill_n(request.correction.x_low, x_size, Real(1)); + }; + auto transition = PreparedAmrProgramRefluxTransition::prepare_with_local_kernel( + coarse, fine, coarse_domain, Periodicity{false, false}, 0, "pops://test/reflux", incomplete, + world_communicator_view()); + + EdgeStrip coarse_role = make_strip(2, 5, 2, 5, 1); + EdgeStrip fine_role = make_strip(2, 5, 2, 5, 1); + coarse_role.cL.assign(4, Real(1)); + coarse_role.cR.assign(4, Real(1)); + coarse_role.cB.assign(4, Real(1)); + coarse_role.cT.assign(4, Real(1)); + fine_role.fL.assign(4, Real(2)); + fine_role.fR.assign(4, Real(2)); + fine_role.fB.assign(4, Real(2)); + fine_role.fT.assign(4, Real(2)); + const amr::ClockStamp logical_time{0, 3, amr::Rational(0, 1), 0.3}; + EXPECT_THROW(transition.synchronize_integrated( + coarse.U, coarse.dx, coarse.dy, std::vector{coarse_role}, + std::vector{fine_role}, world_communicator_view(), &logical_time), + std::runtime_error); + + ASSERT_EQ(coarse.U.local_size(), 1); + EXPECT_EQ(coarse.U.fab(0)(1, 3, 0), Real(0)); + EXPECT_EQ(coarse.U.fab(0)(6, 3, 0), Real(0)); + EXPECT_EQ(coarse.U.fab(0)(3, 1, 0), Real(0)); + EXPECT_EQ(coarse.U.fab(0)(3, 6, 0), Real(0)); +} + TEST(test_program_reflux_ledger, edge_flux_axpy_rejects_shifted_equal_width_footprints) { EdgeFlux destination; destination.fine.push_back(make_strip(2, 5, 2, 5, 1)); From 027c8bba30fa763aa8e90591c3a1a0021709638d Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:07:33 +0200 Subject: [PATCH 065/109] fix(runtime): make field view launch contracts exact (ADC-683) --- .../pops/runtime/config/platform_manifest.hpp | 64 +++++++-- python/pops/_platform_contracts.py | 132 ++++++++++++++++-- 2 files changed, 174 insertions(+), 22 deletions(-) diff --git a/include/pops/runtime/config/platform_manifest.hpp b/include/pops/runtime/config/platform_manifest.hpp index e83c80e4e..177e277ae 100644 --- a/include/pops/runtime/config/platform_manifest.hpp +++ b/include/pops/runtime/config/platform_manifest.hpp @@ -225,12 +225,12 @@ inline void require_same(const std::string& field, const CapabilityProof& expect throw ContractError(field, field + " mismatch between artifact and runtime backend"); } -inline const CapabilityProof& capability(const RuntimeBackendManifest& backend, - const std::string& name) { - const auto found = backend.capabilities.find(name); - if (found == backend.capabilities.end()) +template +inline const CapabilityProof& capability(const Manifest& manifest, const std::string& name) { + const auto found = manifest.capabilities.find(name); + if (found == manifest.capabilities.end()) throw ContractError("capabilities." + name, - "runtime backend omitted required capability proof " + name); + "platform/runtime manifest omitted required capability proof " + name); return found->second; } @@ -248,6 +248,18 @@ inline void validate_descriptor(const FieldViewDescriptor& view) { if (std::any_of(view.ghosts.begin(), view.ghosts.end(), [](const auto& pair) { return pair.first < 0 || pair.second < 0; })) throw ContractError("field.ghosts", "field ghost widths must be non-negative"); + for (std::size_t axis = 0; axis < rank; ++axis) { + const auto lower = static_cast(view.ghosts[axis].first); + const auto upper = static_cast(view.ghosts[axis].second); + if (lower >= view.extents[axis] || upper >= view.extents[axis] - lower) + throw ContractError("field.ghosts", + "field ghost widths must leave a positive interior extent"); + } + if (view.centering.empty() || view.scalar.empty() || view.memory_space.empty() || + view.patch.empty() || view.layout.empty() || view.ownership.empty()) + throw ContractError("field.metadata", + "field centering, scalar, memory space, patch, layout and ownership " + "must be non-empty"); } template @@ -281,33 +293,67 @@ inline void validate_launch(const PlatformManifest& platform, const ExecutionCon !context.device.has_handle) throw ContractError("device", "non-host execution requires an explicit handle"); + for (const std::string name : + {"dimensions", "centerings", "scalars", "layouts", "ownership", "generic_field_view"}) + require_same("capabilities." + name, capability(platform, name), capability(backend, name)); + const auto& generic_field_view = + require(capability(backend, "generic_field_view"), "runtime.capabilities.generic_field_view"); + if (generic_field_view.kind() != CanonicalValue::Kind::kBool || !generic_field_view.boolean()) + throw ContractError("generic_field_view", + "runtime does not prove the generic field-view launch contract"); const auto dimensions = require_int_set(capability(backend, "dimensions"), "runtime.capabilities.dimensions"); const auto centerings = require_text_set(capability(backend, "centerings"), "runtime.capabilities.centerings"); const auto scalars = require_text_set(capability(backend, "scalars"), "runtime.capabilities.scalars"); + const auto layouts = + require_text_set(capability(backend, "layouts"), "runtime.capabilities.layouts"); + const auto ownership = + require_text_set(capability(backend, "ownership"), "runtime.capabilities.ownership"); const auto memories = require_text_set(backend.memory_spaces, "runtime.memory_spaces"); - for (const auto& view : fields) { + std::vector field_names; + field_names.reserve(fields.size()); + std::vector expected_names; + expected_names.reserve(expected.size()); + const auto validate_unique_name = [](const FieldViewDescriptor& view, + std::vector& names, const std::string& owner) { + if (std::find(names.begin(), names.end(), view.name) != names.end()) + throw ContractError("field." + view.name, + owner + " field descriptors must have unique names"); + names.push_back(view.name); + }; + const auto validate_capabilities = [&](const FieldViewDescriptor& view) { validate_descriptor(view); require_member("dimension", view.dimension, dimensions); require_member("centering", view.centering, centerings); require_member("scalar", view.scalar, scalars); require_member("memory_space", view.memory_space, memories); + require_member("layout", view.layout, layouts); + require_member("ownership", view.ownership, ownership); + }; + for (const auto& view : fields) { + validate_unique_name(view, field_names, "launch"); + validate_capabilities(view); if (view.scalar != context.datatype.identity) throw ContractError("datatype", "field scalar and ExecutionContext datatype differ"); const auto wanted = std::find_if(expected.begin(), expected.end(), [&](const auto& item) { return item.name == view.name; }); if (wanted != expected.end() && (view.dimension != wanted->dimension || view.extents != wanted->extents || - view.centering != wanted->centering || view.scalar != wanted->scalar || - view.memory_space != wanted->memory_space)) + view.strides != wanted->strides || view.centering != wanted->centering || + view.ghosts != wanted->ghosts || view.scalar != wanted->scalar || + view.memory_space != wanted->memory_space || view.patch != wanted->patch || + view.layout != wanted->layout || view.ownership != wanted->ownership)) throw ContractError("field." + view.name, "field descriptor does not match launch contract"); } - for (const auto& wanted : expected) + for (const auto& wanted : expected) { + validate_unique_name(wanted, expected_names, "expected"); + validate_capabilities(wanted); if (std::none_of(fields.begin(), fields.end(), [&](const auto& view) { return view.name == wanted.name; })) throw ContractError("field." + wanted.name, "required field descriptor is missing"); + } } template diff --git a/python/pops/_platform_contracts.py b/python/pops/_platform_contracts.py index 9539fa6a4..bffca5e75 100644 --- a/python/pops/_platform_contracts.py +++ b/python/pops/_platform_contracts.py @@ -27,6 +27,26 @@ _CENTERINGS = frozenset({"cell", "node", "face_x", "face_y", "face_z"}) _LAYOUTS = frozenset({"right", "left", "strided"}) _OWNERSHIP = frozenset({"borrowed", "owned", "shared"}) +_FIELD_CAPABILITIES = ( + "dimensions", + "centerings", + "scalars", + "layouts", + "ownership", + "generic_field_view", +) +_EXACT_FIELD_ATTRIBUTES = ( + "dimension", + "extents", + "strides", + "centering", + "ghosts", + "scalar", + "memory_space", + "patch", + "layout", + "ownership", +) _STD_YEARS = {"11": "201103", "14": "201402", "17": "201703", "20": "202002", "23": "202302"} @@ -324,6 +344,10 @@ def __post_init__(self) -> None: len(pair) != 2 or any(isinstance(item, bool) or not isinstance(item, int) or item < 0 for item in pair) for pair in ghosts): raise ValueError("FieldViewDescriptor.ghosts must contain one non-negative pair per axis") + if any(lower >= extent or upper >= extent - lower + for extent, (lower, upper) in zip(self.extents, ghosts, strict=True)): + raise ValueError( + "FieldViewDescriptor.ghosts must leave a positive interior extent on every axis") object.__setattr__(self, "ghosts", ghosts) if self.centering not in _CENTERINGS: raise ValueError("unsupported field centering %r" % self.centering) @@ -411,35 +435,71 @@ def _validate_launch_facts(platform: PlatformManifest, context: ExecutionContext for name in ("storage", "compute", "accumulation", "reduction"): _require_same("precision.%s" % name, getattr(platform.precision, name), getattr(backend.precision, name)) - supported_dimensions = tuple(backend.capabilities["dimensions"].require( + for name in _FIELD_CAPABILITIES: + _require_same( + "capabilities.%s" % name, + _field_capability(platform, name, owner="artifact"), + _field_capability(backend, name, owner="runtime"), + ) + generic_field_view = _field_capability( + backend, "generic_field_view", owner="runtime").require( + "runtime.capabilities.generic_field_view") + if type(generic_field_view) is not bool or not generic_field_view: + raise PlatformContractError( + "runtime does not prove the generic field-view launch contract", + field="generic_field_view", expected=True, actual=generic_field_view) + supported_dimensions = tuple(_field_capability( + backend, "dimensions", owner="runtime").require( "runtime.capabilities.dimensions")) - supported_centerings = tuple(backend.capabilities["centerings"].require( + supported_centerings = tuple(_field_capability( + backend, "centerings", owner="runtime").require( "runtime.capabilities.centerings")) - supported_scalars = tuple(backend.capabilities["scalars"].require( + supported_scalars = tuple(_field_capability( + backend, "scalars", owner="runtime").require( "runtime.capabilities.scalars")) + supported_layouts = tuple(_field_capability( + backend, "layouts", owner="runtime").require( + "runtime.capabilities.layouts")) + supported_ownership = tuple(_field_capability( + backend, "ownership", owner="runtime").require( + "runtime.capabilities.ownership")) supported_memory = tuple(backend.memory_spaces.require("runtime.memory_spaces")) actual = tuple(fields) - expected = {item.name: item for item in expected_fields} - if len(expected) != len(tuple(expected_fields)): - raise ValueError("expected field names must be unique") + required = tuple(expected_fields) + _require_unique_field_names(actual, owner="launch") + _require_unique_field_names(required, owner="expected") + expected = {item.name: item for item in required} for view in actual: - if type(view) is not FieldViewDescriptor: - raise TypeError("fields must contain exact FieldViewDescriptor values") - _require_field_capability(view, "dimension", view.dimension, supported_dimensions) - _require_field_capability(view, "centering", view.centering, supported_centerings) - _require_field_capability(view, "scalar", view.scalar, supported_scalars) - _require_field_capability(view, "memory_space", view.memory_space, supported_memory) + _validate_field_capabilities( + view, + dimensions=supported_dimensions, + centerings=supported_centerings, + scalars=supported_scalars, + memory_spaces=supported_memory, + layouts=supported_layouts, + ownership=supported_ownership, + ) if view.scalar != context.datatype.identity: raise PlatformContractError( "field scalar does not match ExecutionContext datatype", field="datatype", expected=view.scalar, actual=context.datatype.identity) requirement = expected.get(view.name) if requirement is not None: - for name in ("dimension", "extents", "centering", "scalar", "memory_space"): + for name in _EXACT_FIELD_ATTRIBUTES: if getattr(view, name) != getattr(requirement, name): raise PlatformContractError( "field %r %s mismatch" % (view.name, name), field=name, expected=getattr(requirement, name), actual=getattr(view, name)) + for view in required: + _validate_field_capabilities( + view, + dimensions=supported_dimensions, + centerings=supported_centerings, + scalars=supported_scalars, + memory_spaces=supported_memory, + layouts=supported_layouts, + ownership=supported_ownership, + ) missing = sorted(set(expected) - {item.name for item in actual}) if missing: raise PlatformContractError("required field view(s) are missing: %s" % missing, @@ -502,6 +562,12 @@ def validate_component_runtime(platform: PlatformManifest, _require_same( "capabilities.%s" % name, platform.capabilities[name], runtime.capabilities[name]) + for name in _FIELD_CAPABILITIES: + _require_same( + "capabilities.%s" % name, + _field_capability(platform, name, owner="component"), + _field_capability(runtime, name, owner="runtime"), + ) expected_abi = platform.abi.require("component.abi") actual_abi = runtime.abi.require("runtime.abi") if expected_abi != actual_abi: @@ -522,6 +588,46 @@ def _require_field_capability(view: FieldViewDescriptor, field_name: str, expected=supported, actual=value) +def _field_capability(manifest: PlatformManifest | RuntimeBackendManifest, name: str, + *, owner: str) -> CapabilityProof: + proof = manifest.capabilities.get(name) + if proof is None: + raise PlatformContractError( + "%s omitted required field-view capability %r" % (owner, name), + field="capabilities.%s" % name, expected="explicit proof", actual=None) + return proof + + +def _require_unique_field_names(fields: tuple[FieldViewDescriptor, ...], *, owner: str) -> None: + names: set[str] = set() + for view in fields: + if type(view) is not FieldViewDescriptor: + raise TypeError("%s fields must contain exact FieldViewDescriptor values" % owner) + if view.name in names: + raise PlatformContractError( + "%s field descriptors contain duplicate name %r" % (owner, view.name), + field="fields.%s" % view.name, expected="unique name", actual=view.name) + names.add(view.name) + + +def _validate_field_capabilities( + view: FieldViewDescriptor, + *, + dimensions: tuple[Any, ...], + centerings: tuple[Any, ...], + scalars: tuple[Any, ...], + memory_spaces: tuple[Any, ...], + layouts: tuple[Any, ...], + ownership: tuple[Any, ...], +) -> None: + _require_field_capability(view, "dimension", view.dimension, dimensions) + _require_field_capability(view, "centering", view.centering, centerings) + _require_field_capability(view, "scalar", view.scalar, scalars) + _require_field_capability(view, "memory_space", view.memory_space, memory_spaces) + _require_field_capability(view, "layout", view.layout, layouts) + _require_field_capability(view, "ownership", view.ownership, ownership) + + def launch_checked(platform: PlatformManifest, context: ExecutionContext, fields: Sequence[FieldViewDescriptor], kernel: Callable[..., Any], *, expected_fields: Sequence[FieldViewDescriptor] = ()) -> Any: From 0ac494211bf91b4ec1ce58497b79d3dd26224cb6 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:07:38 +0200 Subject: [PATCH 066/109] test(runtime): fence complete field view descriptors (ADC-683) --- .../unit/runtime/test_platform_manifest.cpp | 65 +++++++++++++++-- .../unit/runtime/test_platform_manifest.py | 72 +++++++++++++++++++ 2 files changed, 133 insertions(+), 4 deletions(-) diff --git a/tests/cpp/unit/runtime/test_platform_manifest.cpp b/tests/cpp/unit/runtime/test_platform_manifest.cpp index 01afeb06f..80c9b0424 100644 --- a/tests/cpp/unit/runtime/test_platform_manifest.cpp +++ b/tests/cpp/unit/runtime/test_platform_manifest.cpp @@ -72,9 +72,8 @@ TEST(PlatformManifest, FieldAndCommunicatorMismatchesRefuseBeforeKernel) { int launches = 0; auto kernel = [&](const auto&, const auto&) { return ++launches; }; const auto required = field(); - for (int variant = 0; variant < 5; ++variant) { + for (int variant = 0; variant < 9; ++variant) { auto actual = field(); - auto execution = context(); if (variant == 0) actual.centering = "node"; else if (variant == 1) @@ -83,13 +82,71 @@ TEST(PlatformManifest, FieldAndCommunicatorMismatchesRefuseBeforeKernel) { actual.extents = {15, 12}; else if (variant == 3) actual.memory_space = "device"; + else if (variant == 4) + actual.strides = {1, 16}; + else if (variant == 5) + actual.ghosts = {{1, 0}, {0, 0}}; + else if (variant == 6) + actual.patch = "patch-1"; + else if (variant == 7) + actual.layout = "left"; else - execution.communicator.identity = "comm:wrong"; + actual.ownership = "owned"; EXPECT_THROW( - pops::platform::launch_checked(platform(), execution, {actual}, kernel, {required}), + pops::platform::launch_checked(platform(), context(), {actual}, kernel, {required}), pops::platform::ContractError); EXPECT_EQ(launches, 0); } + auto execution = context(); + execution.communicator.identity = "comm:wrong"; + EXPECT_THROW(pops::platform::launch_checked(platform(), execution, {field()}, kernel, {required}), + pops::platform::ContractError); + EXPECT_EQ(launches, 0); +} + +TEST(PlatformManifest, FieldCapabilitiesAndNamesFailClosed) { + int launches = 0; + auto kernel = [&](const auto&, const auto&) { return ++launches; }; + + auto missing = platform(); + missing.capabilities.erase("ownership"); + EXPECT_THROW(pops::platform::launch_checked(missing, context(), {field()}, kernel), + pops::platform::ContractError); + + auto unsupported = platform(); + unsupported.capabilities["layouts"] = pops::platform::prove_text_set({"left"}, "test"); + auto unsupported_context = context(); + unsupported_context.backend.capabilities["layouts"] = + pops::platform::prove_text_set({"left"}, "test"); + EXPECT_THROW(pops::platform::launch_checked(unsupported, unsupported_context, {field()}, kernel), + pops::platform::ContractError); + + auto disabled = platform(); + disabled.capabilities["generic_field_view"] = pops::platform::prove_bool(false, "test"); + auto disabled_context = context(); + disabled_context.backend.capabilities["generic_field_view"] = + pops::platform::prove_bool(false, "test"); + EXPECT_THROW(pops::platform::launch_checked(disabled, disabled_context, {field()}, kernel), + pops::platform::ContractError); + + EXPECT_THROW(pops::platform::launch_checked(platform(), context(), {field(), field()}, kernel), + pops::platform::ContractError); + EXPECT_THROW( + pops::platform::launch_checked(platform(), context(), {field()}, kernel, {field(), field()}), + pops::platform::ContractError); + EXPECT_EQ(launches, 0); +} + +TEST(PlatformManifest, FieldGhostsMustLeavePositiveInterior) { + auto hidden = field(); + hidden.ghosts = {{16, 0}, {0, 0}}; + EXPECT_THROW(pops::platform::validate_launch(platform(), context(), {hidden}), + pops::platform::ContractError); + + hidden = field(); + hidden.ghosts = {{8, 8}, {0, 0}}; + EXPECT_THROW(pops::platform::validate_launch(platform(), context(), {hidden}), + pops::platform::ContractError); } TEST(PlatformManifest, GenericTwoDimensionalDoubleRouteLaunches) { diff --git a/tests/python/unit/runtime/test_platform_manifest.py b/tests/python/unit/runtime/test_platform_manifest.py index 135559457..a24290307 100644 --- a/tests/python/unit/runtime/test_platform_manifest.py +++ b/tests/python/unit/runtime/test_platform_manifest.py @@ -14,6 +14,7 @@ launch_checked, proven_serial_manifest, validate_component_launch, + validate_component_runtime, validate_launch, ) from pops.identity import make_identity @@ -125,6 +126,11 @@ def test_unknown_is_missing_proof_and_3d_is_representable_then_refused(): {"scalar": "float32"}, {"extents": (15, 12)}, {"memory_space": "device"}, + {"strides": (1, 16)}, + {"ghosts": ((1, 0), (0, 0))}, + {"patch": "patch-1"}, + {"layout": "left"}, + {"ownership": "owned"}, ]) def test_field_mismatch_refuses_before_kernel(changed): launched = [] @@ -135,6 +141,62 @@ def test_field_mismatch_refuses_before_kernel(changed): assert launched == [] +def test_field_view_requires_exact_capability_proofs_before_kernel(): + launched = [] + platform = _platform() + context = _context() + + missing = dict(platform.capabilities) + missing.pop("ownership") + with pytest.raises(PlatformContractError, match="omitted required field-view capability"): + launch_checked( + replace(platform, capabilities=missing), context, [_field()], + lambda *_: launched.append(True)) + + unsupported_layout = _proof(("left",)) + artifact_capabilities = dict(platform.capabilities, layouts=unsupported_layout) + runtime_capabilities = dict(context.backend.capabilities, layouts=unsupported_layout) + with pytest.raises(PlatformContractError, match="unsupported layout='right'"): + launch_checked( + replace(platform, capabilities=artifact_capabilities), + replace(context, backend=replace( + context.backend, capabilities=runtime_capabilities)), + [_field()], lambda *_: launched.append(True)) + + generic_disabled = _proof(False) + artifact_capabilities = dict(platform.capabilities, generic_field_view=generic_disabled) + runtime_capabilities = dict(context.backend.capabilities, generic_field_view=generic_disabled) + with pytest.raises(PlatformContractError, match="does not prove the generic field-view"): + launch_checked( + replace(platform, capabilities=artifact_capabilities), + replace(context, backend=replace( + context.backend, capabilities=runtime_capabilities)), + [_field()], lambda *_: launched.append(True)) + + assert launched == [] + + +@pytest.mark.parametrize("expected", [False, True]) +def test_duplicate_field_names_refuse_before_kernel(expected): + launched = [] + actual_fields = [_field(), _field()] + expected_fields = [_field(), _field()] if expected else [_field()] + if expected: + actual_fields = [_field()] + with pytest.raises(PlatformContractError, match="descriptors contain duplicate name"): + launch_checked( + _platform(), _context(), actual_fields, lambda *_: launched.append(True), + expected_fields=expected_fields) + assert launched == [] + + +def test_field_view_ghosts_must_leave_positive_interior(): + with pytest.raises(ValueError, match="positive interior"): + _field(ghosts=((16, 0), (0, 0))) + with pytest.raises(ValueError, match="positive interior"): + _field(ghosts=((8, 8), (0, 0))) + + def test_generic_2d_double_descriptor_launches_once(): launched = [] assert launch_checked( @@ -163,6 +225,16 @@ def test_aot_component_build_route_is_checked_against_simulation_execution_facts validate_component_launch(_platform(), context, ()) +def test_aot_component_field_capabilities_fail_before_native_load(): + component = proven_serial_manifest( + backend="aot-component", target="component", abi="headers|clang|c++23") + runtime = _context().backend + missing = dict(component.capabilities) + missing.pop("layouts") + with pytest.raises(PlatformContractError, match="omitted required field-view capability"): + validate_component_runtime(replace(component, capabilities=missing), runtime) + + def test_aot_component_rejects_openmpi_mpich_abi_mix_even_with_same_headers_and_standard(): openmpi = ( "compiler=clang;std=202002;headers=same;kokkos=1;stdlib=libc++;" From 40a0b070e0d23f3ac523b0f06c76f302ef175f17 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:16:56 +0200 Subject: [PATCH 067/109] refactor(output): require observer-owned HDF5 communication (ADC-683) --- .../pops/runtime/output/hdf5_collective.hpp | 10 - .../bindings/core/init/init_parallel_hdf5.cpp | 22 +- python/pops/output/_writers/hdf5.py | 2 +- src/runtime/output/hdf5_collective.cpp | 195 ++++++++---------- 4 files changed, 90 insertions(+), 139 deletions(-) diff --git a/include/pops/runtime/output/hdf5_collective.hpp b/include/pops/runtime/output/hdf5_collective.hpp index 9a5e4bf51..c89db165b 100644 --- a/include/pops/runtime/output/hdf5_collective.hpp +++ b/include/pops/runtime/output/hdf5_collective.hpp @@ -6,10 +6,6 @@ #include -namespace pops { -class WorldCommunicator; -} - namespace pops::runtime::output { /// Non-owning, contiguous NumPy-compatible array view used by the native HDF5 adapter. @@ -56,8 +52,6 @@ struct ParallelHdf5Capability { /// rank is allowed to enter HDF5. An empty string means that local validation succeeded. void collective_hdf5_input_consensus(const CommunicatorView& communicator, const std::string& local_error); -void collective_hdf5_input_consensus(const WorldCommunicator& world, - const std::string& local_error); /// Write one exact scientific-output artifact collectively on an explicit native communicator. /// @@ -71,9 +65,5 @@ void write_collective_hdf5(const CommunicatorView& communicator, const std::stri const std::string& manifest_json, const std::vector& root_arrays, const std::vector& fields); -void write_collective_hdf5(const WorldCommunicator& world, const std::string& path, - const std::string& manifest_json, - const std::vector& root_arrays, - const std::vector& fields); } // namespace pops::runtime::output diff --git a/python/bindings/core/init/init_parallel_hdf5.cpp b/python/bindings/core/init/init_parallel_hdf5.cpp index d2427257c..5e8cf2bde 100644 --- a/python/bindings/core/init/init_parallel_hdf5.cpp +++ b/python/bindings/core/init/init_parallel_hdf5.cpp @@ -1,7 +1,6 @@ #include "../bindings_detail.hpp" #include -#include #include #include @@ -97,21 +96,12 @@ void init_parallel_hdf5(py::module_& m) { [](const py::object& communicator_value, const py::object& path_value, const py::object& manifest_value, const py::object& root_arrays_value, const py::object& field_rows_value) { - pops::CommunicatorView communicator; - if (py::isinstance(communicator_value)) { - auto& world = communicator_value.cast(); - if (&world != &pops::WorldCommunicator::world()) - throw py::value_error("native HDF5 requires the exact process-world authority"); - communicator = world.communicator(); - } else if (py::isinstance(communicator_value)) { - auto& lane = communicator_value.cast(); - if (!lane.active()) - throw py::value_error("native HDF5 observer lane is closed"); - communicator = lane.communicator(); - } else { - throw py::type_error( - "native HDF5 requires a PoPS world communicator or observer MPI lane"); - } + if (!py::isinstance(communicator_value)) + throw py::type_error("native HDF5 requires an exact duplicated observer MPI lane"); + auto& lane = communicator_value.cast(); + if (!lane.active()) + throw py::value_error("native HDF5 observer lane is closed"); + const pops::CommunicatorView communicator = lane.communicator(); std::vector owners; std::vector arrays; std::vector fields; diff --git a/python/pops/output/_writers/hdf5.py b/python/pops/output/_writers/hdf5.py index 0e5c88609..6356eed6b 100644 --- a/python/pops/output/_writers/hdf5.py +++ b/python/pops/output/_writers/hdf5.py @@ -177,7 +177,7 @@ def _parallel_snapshot_data( if request.parallel_mode is not ParallelMode.COLLECTIVE: raise ValueError( "a resolved communicator is valid only for HDF5 COLLECTIVE output") - require_communicator(communicator) + require_communicator(communicator, allow_world=False) if request.rank != rank(communicator): raise ValueError("collective HDF5 request rank differs from its native communicator") native, capability = _require_native_parallel_hdf5() diff --git a/src/runtime/output/hdf5_collective.cpp b/src/runtime/output/hdf5_collective.cpp index 32781070d..aaa84e31c 100644 --- a/src/runtime/output/hdf5_collective.cpp +++ b/src/runtime/output/hdf5_collective.cpp @@ -1,5 +1,4 @@ #include -#include #include #include @@ -160,9 +159,9 @@ template template [[nodiscard]] AgreedFailure collective_phase(int rank, MPI_Comm communicator, - Operation&& operation) { - return agree_failure( - rank, capture_local_failure(std::forward(operation)), communicator); + Operation&& operation) { + return agree_failure(rank, capture_local_failure(std::forward(operation)), + communicator); } [[noreturn]] void throw_collective_failure(std::string_view phase, std::string_view subject, @@ -186,8 +185,8 @@ void require_collective_success(std::string_view phase, std::string_view subject [[nodiscard]] AgreedFailure require_identical_text(int rank, std::string_view local, MPI_Comm communicator) { - int overflow = local.size() > static_cast( - std::numeric_limits::max()); + int overflow = + local.size() > static_cast(std::numeric_limits::max()); require_mpi(MPI_Allreduce(MPI_IN_PLACE, &overflow, 1, MPI_INT, MPI_MAX, communicator), "MPI_Allreduce(schema length overflow)"); if (overflow != 0) { @@ -217,8 +216,7 @@ void require_collective_success(std::string_view phase, std::string_view subject length - offset, static_cast(std::numeric_limits::max()))); char* buffer = rank == 0 ? const_cast(local.data()) + static_cast(offset) : reference.data() + static_cast(offset); - require_mpi(MPI_Bcast(buffer, count, MPI_CHAR, 0, communicator), - "MPI_Bcast(schema bytes)"); + require_mpi(MPI_Bcast(buffer, count, MPI_CHAR, 0, communicator), "MPI_Bcast(schema bytes)"); offset += static_cast(count); } @@ -231,8 +229,7 @@ void require_collective_success(std::string_view phase, std::string_view subject using PieceDescriptor = std::array; -[[nodiscard]] std::vector piece_descriptors( - const std::vector& fields) { +[[nodiscard]] std::vector piece_descriptors(const std::vector& fields) { std::size_t count = 0; for (const auto& field : fields) count = checked_add(count, field.pieces.size(), "native HDF5 piece descriptor count"); @@ -251,11 +248,10 @@ using PieceDescriptor = std::array; } } for (const auto& piece : field.pieces) { - result.push_back({static_cast(field_index), - static_cast(piece.jlo), - static_cast(piece.ilo), - static_cast(piece.jhi), - static_cast(piece.ihi)}); + result.push_back( + {static_cast(field_index), static_cast(piece.jlo), + static_cast(piece.ilo), static_cast(piece.jhi), + static_cast(piece.ihi)}); } } return result; @@ -263,13 +259,14 @@ using PieceDescriptor = std::array; [[nodiscard]] bool pieces_overlap(const PieceDescriptor& left, const PieceDescriptor& right) noexcept { - return left[0] == right[0] && left[1] < right[3] && right[1] < left[3] && - left[2] < right[4] && right[2] < left[4]; + return left[0] == right[0] && left[1] < right[3] && right[1] < left[3] && left[2] < right[4] && + right[2] < left[4]; } -[[nodiscard]] AgreedFailure require_disjoint_rank_pieces( - int rank, int ranks, const std::vector& local, - const std::vector& fields, MPI_Comm communicator) { +[[nodiscard]] AgreedFailure require_disjoint_rank_pieces(int rank, int ranks, + const std::vector& local, + const std::vector& fields, + MPI_Comm communicator) { static_assert(sizeof(PieceDescriptor) == 5 * sizeof(unsigned long long)); int length_overflow = 0; if constexpr (sizeof(std::size_t) > sizeof(unsigned long long)) { @@ -301,8 +298,7 @@ using PieceDescriptor = std::array; return finish(type_failure); for (int owner = 0; owner < ranks; ++owner) { - unsigned long long count = - rank == owner ? static_cast(local.size()) : 0ULL; + unsigned long long count = rank == owner ? static_cast(local.size()) : 0ULL; require_mpi(MPI_Bcast(&count, 1, MPI_UNSIGNED_LONG_LONG, owner, communicator), "MPI_Bcast(piece descriptor count)"); @@ -321,8 +317,8 @@ using PieceDescriptor = std::array; while (offset < count) { const int chunk = static_cast(std::min( count - offset, static_cast(std::numeric_limits::max()))); - require_mpi(MPI_Bcast(buffer + static_cast(offset), chunk, descriptor_type, owner, - communicator), + require_mpi(MPI_Bcast(buffer + static_cast(offset), chunk, descriptor_type, + owner, communicator), "MPI_Bcast(piece descriptors)"); offset += static_cast(chunk); } @@ -335,12 +331,12 @@ using PieceDescriptor = std::array; if (!pieces_overlap(mine, theirs)) continue; const auto field_index = static_cast(mine[0]); - const std::string_view dataset = - field_index < fields.size() ? std::string_view{fields[field_index].dataset} - : std::string_view{""}; - throw std::invalid_argument( - "field pieces overlap across MPI ranks " + std::to_string(owner) + " and " + - std::to_string(rank) + " for dataset " + std::string(dataset)); + const std::string_view dataset = field_index < fields.size() + ? std::string_view{fields[field_index].dataset} + : std::string_view{""}; + throw std::invalid_argument("field pieces overlap across MPI ranks " + + std::to_string(owner) + " and " + std::to_string(rank) + + " for dataset " + std::string(dataset)); } } }); @@ -566,8 +562,7 @@ void validate_inputs(const std::string& path, const std::string& manifest, return result; } -[[nodiscard]] std::vector group_paths( - const std::vector& datasets) { +[[nodiscard]] std::vector group_paths(const std::vector& datasets) { std::vector groups; for (const auto& dataset : datasets) { std::size_t cursor = 0; @@ -590,8 +585,8 @@ struct DatasetCreatePlan { std::vector zero; }; -[[nodiscard]] DatasetCreatePlan prepare_dataset_creation( - const std::vector& shape, const std::string& dtype) { +[[nodiscard]] DatasetCreatePlan prepare_dataset_creation(const std::vector& shape, + const std::string& dtype) { const auto dimensions = hdf5_shape(shape); DatasetCreatePlan plan; plan.space = H5Handle( @@ -766,12 +761,9 @@ struct ManifestAttributePlan { ParallelHdf5Capability parallel_hdf5_capability() { #if defined(POPS_HAS_PARALLEL_HDF5) - std::lock_guard guard{parallel_hdf5_mutex()}; - int initialized = 0; - require_mpi(MPI_Initialized(&initialized), "MPI_Initialized"); const std::string version = std::to_string(H5_VERS_MAJOR) + "." + std::to_string(H5_VERS_MINOR) + "." + std::to_string(H5_VERS_RELEASE); - return {true, version, initialized ? "" : "MPI is compiled but not initialized"}; + return {true, version, ""}; #else return {false, "", "module was not built with MPI and a parallel HDF5 C library"}; #endif @@ -794,19 +786,10 @@ void collective_hdf5_input_consensus(const CommunicatorView& communicator, LocalFailure local; if (!local_error.empty()) set_failure(local, local_error); - require_collective_success( - "binding input validation", "", agree_failure(rank, local, native)); + require_collective_success("binding input validation", "", agree_failure(rank, local, native)); #endif } -void collective_hdf5_input_consensus(const WorldCommunicator& world, - const std::string& local_error) { - if (&world != &WorldCommunicator::world()) - throw std::invalid_argument( - "collective HDF5 requires the exact native process-world authority"); - collective_hdf5_input_consensus(world.communicator(), local_error); -} - void write_collective_hdf5(const CommunicatorView& communicator, const std::string& path, const std::string& manifest_json, const std::vector& root_arrays, @@ -831,24 +814,22 @@ void write_collective_hdf5(const CommunicatorView& communicator, const std::stri require_mpi(MPI_Comm_size(native, &ranks), "MPI_Comm_size"); require_collective_success("input validation", "", collective_phase(rank, native, [&] { - validate_inputs(path, manifest_json, root_arrays, fields); - })); + validate_inputs(path, manifest_json, root_arrays, fields); + })); std::string schema; require_collective_success("schema preparation", "", collective_phase(rank, native, [&] { - schema = schema_text(path, manifest_json, root_arrays, fields); - })); - require_collective_success( - "schema consensus", "", require_identical_text(rank, schema, native)); + schema = schema_text(path, manifest_json, root_arrays, fields); + })); + require_collective_success("schema consensus", "", require_identical_text(rank, schema, native)); std::vector descriptors; require_collective_success( - "piece descriptor preparation", "", collective_phase(rank, native, [&] { - descriptors = piece_descriptors(fields); - })); - require_collective_success("piece descriptor consensus", "", - require_disjoint_rank_pieces( - rank, ranks, descriptors, fields, native)); + "piece descriptor preparation", "", + collective_phase(rank, native, [&] { descriptors = piece_descriptors(fields); })); + require_collective_success( + "piece descriptor consensus", "", + require_disjoint_rank_pieces(rank, ranks, descriptors, fields, native)); std::vector dataset_names; std::vector groups; @@ -861,48 +842,49 @@ void write_collective_hdf5(const CommunicatorView& communicator, const std::stri H5Handle transfer; require_collective_success( "local HDF5 preparation", "", collective_phase(rank, native, [&] { - dataset_names.reserve(root_arrays.size() + fields.size()); - for (const auto& array : root_arrays) - dataset_names.push_back(array.dataset); - for (const auto& field : fields) - dataset_names.push_back(field.dataset); - groups = group_paths(dataset_names); - - root_creation_plans.reserve(root_arrays.size()); - for (const auto& array : root_arrays) - root_creation_plans.push_back( - prepare_dataset_creation(array.values.shape, array.values.dtype)); - field_creation_plans.reserve(fields.size()); - for (const auto& field : fields) - field_creation_plans.push_back(prepare_dataset_creation(field.shape, field.dtype)); - manifest_plan = prepare_manifest_attribute(); - group_creation = H5Handle(H5Pcreate(H5P_GROUP_CREATE), H5Pclose); - if (!group_creation || H5Pset_obj_track_times(group_creation.get(), false) < 0) - throw std::runtime_error("HDF5 deterministic group creation-property preparation failed"); - file_creation = H5Handle(H5Pcreate(H5P_FILE_CREATE), H5Pclose); - if (!file_creation || H5Pset_obj_track_times(file_creation.get(), false) < 0) - throw std::runtime_error("HDF5 deterministic file creation-property preparation failed"); - - access = H5Handle(H5Pcreate(H5P_FILE_ACCESS), H5Pclose); - if (!access || H5Pset_fapl_mpio(access.get(), native, MPI_INFO_NULL) < 0) - throw std::runtime_error("H5Pset_fapl_mpio(explicit communicator) failed"); + dataset_names.reserve(root_arrays.size() + fields.size()); + for (const auto& array : root_arrays) + dataset_names.push_back(array.dataset); + for (const auto& field : fields) + dataset_names.push_back(field.dataset); + groups = group_paths(dataset_names); + + root_creation_plans.reserve(root_arrays.size()); + for (const auto& array : root_arrays) + root_creation_plans.push_back( + prepare_dataset_creation(array.values.shape, array.values.dtype)); + field_creation_plans.reserve(fields.size()); + for (const auto& field : fields) + field_creation_plans.push_back(prepare_dataset_creation(field.shape, field.dtype)); + manifest_plan = prepare_manifest_attribute(); + group_creation = H5Handle(H5Pcreate(H5P_GROUP_CREATE), H5Pclose); + if (!group_creation || H5Pset_obj_track_times(group_creation.get(), false) < 0) + throw std::runtime_error("HDF5 deterministic group creation-property preparation failed"); + file_creation = H5Handle(H5Pcreate(H5P_FILE_CREATE), H5Pclose); + if (!file_creation || H5Pset_obj_track_times(file_creation.get(), false) < 0) + throw std::runtime_error("HDF5 deterministic file creation-property preparation failed"); + + access = H5Handle(H5Pcreate(H5P_FILE_ACCESS), H5Pclose); + if (!access || H5Pset_fapl_mpio(access.get(), native, MPI_INFO_NULL) < 0) + throw std::runtime_error("H5Pset_fapl_mpio(explicit communicator) failed"); #if H5_VERSION_GE(1, 10, 0) - if (H5Pset_all_coll_metadata_ops(access.get(), 1) < 0 || - H5Pset_coll_metadata_write(access.get(), 1) < 0) - throw std::runtime_error("parallel HDF5 collective metadata configuration failed"); + if (H5Pset_all_coll_metadata_ops(access.get(), 1) < 0 || + H5Pset_coll_metadata_write(access.get(), 1) < 0) + throw std::runtime_error("parallel HDF5 collective metadata configuration failed"); #endif - transfer = H5Handle(H5Pcreate(H5P_DATASET_XFER), H5Pclose); - if (!transfer || H5Pset_dxpl_mpio(transfer.get(), H5FD_MPIO_COLLECTIVE) < 0) - throw std::runtime_error("H5Pset_dxpl_mpio(H5FD_MPIO_COLLECTIVE) failed"); - })); + transfer = H5Handle(H5Pcreate(H5P_DATASET_XFER), H5Pclose); + if (!transfer || H5Pset_dxpl_mpio(transfer.get(), H5FD_MPIO_COLLECTIVE) < 0) + throw std::runtime_error("H5Pset_dxpl_mpio(H5FD_MPIO_COLLECTIVE) failed"); + })); H5Handle file; - require_collective_success("file creation", path, collective_phase(rank, native, [&] { - file = H5Handle(H5Fcreate(path.c_str(), H5F_ACC_TRUNC, file_creation.get(), access.get()), - H5Fclose); - if (!file) - throw std::runtime_error("H5Fcreate returned an invalid handle"); - })); + require_collective_success( + "file creation", path, collective_phase(rank, native, [&] { + file = H5Handle(H5Fcreate(path.c_str(), H5F_ACC_TRUNC, file_creation.get(), access.get()), + H5Fclose); + if (!file) + throw std::runtime_error("H5Fcreate returned an invalid handle"); + })); AgreedFailure transaction_failure; auto remember_failure = [&](const AgreedFailure& failure) noexcept { @@ -1028,24 +1010,13 @@ void write_collective_hdf5(const CommunicatorView& communicator, const std::stri } const auto close_failure = collective_phase(rank, native, [&] { - const hid_t handle = file.release(); - if (H5Fclose(handle) < 0) - throw std::runtime_error("H5Fclose failed"); + const hid_t handle = file.release(); + if (H5Fclose(handle) < 0) + throw std::runtime_error("H5Fclose failed"); }); remember_failure(close_failure); require_collective_success("transaction", "", transaction_failure); #endif } -void write_collective_hdf5(const WorldCommunicator& world, const std::string& path, - const std::string& manifest_json, - const std::vector& root_arrays, - const std::vector& fields) { - if (&world != &WorldCommunicator::world()) - throw std::invalid_argument( - "collective HDF5 requires the exact native process-world authority"); - write_collective_hdf5( - world.communicator(), path, manifest_json, root_arrays, fields); -} - } // namespace pops::runtime::output From c32d76ec02324192f95639cb287c696484b32c96 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:17:26 +0200 Subject: [PATCH 068/109] test(output): fence observer-owned HDF5 lanes (ADC-683) --- .../mpi/test_mpi_hdf5_collective.cpp | 14 ++++-- .../test_hdf5_observer_lane_fence.py | 33 ++++++++++++ .../integration/io/test_hdf5_parallel.py | 50 ++++++++++++------- .../mpi/test_scientific_output_mpi.py | 35 +++++++------ 4 files changed, 93 insertions(+), 39 deletions(-) create mode 100644 tests/python/architecture/test_hdf5_observer_lane_fence.py diff --git a/tests/cpp/integration/mpi/test_mpi_hdf5_collective.cpp b/tests/cpp/integration/mpi/test_mpi_hdf5_collective.cpp index 615d33c52..8f8a43df2 100644 --- a/tests/cpp/integration/mpi/test_mpi_hdf5_collective.cpp +++ b/tests/cpp/integration/mpi/test_mpi_hdf5_collective.cpp @@ -139,6 +139,7 @@ TEST(MpiHdf5Collective, WritesDisjointHyperslabsAndReopensNatively) { FAIL() << "this target must never be registered without native parallel HDF5"; #else auto& world = pops::WorldCommunicator::world(); + const auto communicator = world.communicator(); const int rank = world.rank(); const int ranks = world.size(); ASSERT_GE(rank, 0); @@ -180,7 +181,7 @@ TEST(MpiHdf5Collective, WritesDisjointHyperslabsAndReopensNatively) { local_values.size() * sizeof(double)}}}, }}; const std::string manifest = R"({"format":"native-test","version":1})"; - pops::runtime::output::write_collective_hdf5(world, path_text, manifest, arrays, fields); + pops::runtime::output::write_collective_hdf5(communicator, path_text, manifest, arrays, fields); std::string validation_error; if (rank == 0) { @@ -203,6 +204,7 @@ TEST(MpiHdf5Collective, RejectsOneRankInvalidDescriptorBeforeCreatingFile) { FAIL() << "this target must never be registered without native parallel HDF5"; #else auto& world = pops::WorldCommunicator::world(); + const auto communicator = world.communicator(); const int rank = world.rank(); const int ranks = world.size(); if (ranks < 2) @@ -240,7 +242,7 @@ TEST(MpiHdf5Collective, RejectsOneRankInvalidDescriptorBeforeCreatingFile) { std::string error; try { pops::runtime::output::write_collective_hdf5( - world, path, R"({"format":"native-invalid-test","version":1})", arrays, fields); + communicator, path, R"({"format":"native-invalid-test","version":1})", arrays, fields); } catch (const std::exception& failure) { error = failure.what(); } @@ -266,6 +268,7 @@ TEST(MpiHdf5Collective, RejectsCrossRankOverlappingHyperslabsBeforeCreatingFile) FAIL() << "this target must never be registered without native parallel HDF5"; #else auto& world = pops::WorldCommunicator::world(); + const auto communicator = world.communicator(); const int rank = world.rank(); const int ranks = world.size(); if (ranks < 2) @@ -303,7 +306,7 @@ TEST(MpiHdf5Collective, RejectsCrossRankOverlappingHyperslabsBeforeCreatingFile) std::string error; try { pops::runtime::output::write_collective_hdf5( - world, path, R"({"format":"native-overlap-test","version":1})", arrays, fields); + communicator, path, R"({"format":"native-overlap-test","version":1})", arrays, fields); } catch (const std::exception& failure) { error = failure.what(); } @@ -329,6 +332,7 @@ TEST(MpiHdf5Collective, RepeatedIdenticalWritesAreByteIdenticalAcrossTime) { FAIL() << "this target must never be registered without native parallel HDF5"; #else auto& world = pops::WorldCommunicator::world(); + const auto communicator = world.communicator(); const int rank = world.rank(); const int ranks = world.size(); const std::string first_path = shared_temporary_path(world, "native-parallel-hdf5-exact-a"); @@ -369,9 +373,9 @@ TEST(MpiHdf5Collective, RepeatedIdenticalWritesAreByteIdenticalAcrossTime) { }}; const std::string manifest = R"({"format":"native-exact-test","version":1})"; - pops::runtime::output::write_collective_hdf5(world, first_path, manifest, arrays, fields); + pops::runtime::output::write_collective_hdf5(communicator, first_path, manifest, arrays, fields); std::this_thread::sleep_for(std::chrono::milliseconds(1200)); - pops::runtime::output::write_collective_hdf5(world, second_path, manifest, arrays, fields); + pops::runtime::output::write_collective_hdf5(communicator, second_path, manifest, arrays, fields); std::string validation_error; if (rank == 0) { diff --git a/tests/python/architecture/test_hdf5_observer_lane_fence.py b/tests/python/architecture/test_hdf5_observer_lane_fence.py new file mode 100644 index 000000000..ee07cd388 --- /dev/null +++ b/tests/python/architecture/test_hdf5_observer_lane_fence.py @@ -0,0 +1,33 @@ +"""ADC-683 fences for observer-owned collective HDF5 communication.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +HEADER = ROOT / "include/pops/runtime/output/hdf5_collective.hpp" +SOURCE = ROOT / "src/runtime/output/hdf5_collective.cpp" +BINDING = ROOT / "python/bindings/core/init/init_parallel_hdf5.cpp" +WRITER = ROOT / "python/pops/output/_writers/hdf5.py" + + +def test_native_hdf5_surface_has_no_process_world_overload_or_probe(): + header = HEADER.read_text(encoding="utf-8") + source = SOURCE.read_text(encoding="utf-8") + + assert "WorldCommunicator" not in header + assert "WorldCommunicator" not in source + assert "world_communicator.hpp" not in source + assert "MPI_COMM_WORLD" not in source + assert "MPI_Initialized" not in source + assert "const CommunicatorView& communicator" in header + + +def test_python_hdf5_route_requires_a_duplicated_observer_lane(): + binding = BINDING.read_text(encoding="utf-8") + writer = WRITER.read_text(encoding="utf-8") + + assert "WorldCommunicator" not in binding + assert "world_communicator.hpp" not in binding + assert "py::isinstance" in binding + assert "requires an exact duplicated observer MPI lane" in binding + assert "require_communicator(communicator, allow_world=False)" in writer diff --git a/tests/python/integration/io/test_hdf5_parallel.py b/tests/python/integration/io/test_hdf5_parallel.py index a5b7c1858..6643a7367 100644 --- a/tests/python/integration/io/test_hdf5_parallel.py +++ b/tests/python/integration/io/test_hdf5_parallel.py @@ -119,15 +119,15 @@ def snapshot(pieces): return snapshot((local_piece,)), snapshot(serial_pieces), key, global_values -def _parallel_hdf5_world(test_name: str): +def _parallel_hdf5_lane(test_name: str): try: import h5py # noqa: F401 -- serial native reopen verification except ImportError: _missing_mpi_requirement("collective HDF5 requires h5py") if getattr(_pops, "__has_parallel_hdf5__", False) is not True: _missing_mpi_requirement("collective HDF5 requires the compiled C++ parallel-HDF5 route") - communicator = _pops.mpi_world() - if world_size(communicator) == 1 and os.environ.get(_MPI_CHILD) != "1": + world = _pops.mpi_world() + if world_size(world) == 1 and os.environ.get(_MPI_CHILD) != "1": mpiexec = shutil.which("mpiexec") or shutil.which("mpirun") if mpiexec is None: _missing_mpi_requirement( @@ -154,12 +154,24 @@ def _parallel_hdf5_world(test_name: str): ) assert result.returncode == 0, result.stdout + result.stderr return None - assert world_size(communicator) >= 2, "MPI child did not start with two ranks" - return communicator + assert world_size(world) >= 2, "MPI child did not start with two ranks" + return world.duplicate_observer_lane("pytest-hdf5-" + test_name) -def test_collective_hdf5_roundtrip_matches_serial(tmp_path): - communicator = _parallel_hdf5_world(test_collective_hdf5_roundtrip_matches_serial.__name__) +@pytest.fixture +def parallel_hdf5_lane(request): + lane = _parallel_hdf5_lane(request.node.name) + if lane is None: + yield None + return + try: + yield lane + finally: + lane.close_collectively() + + +def test_collective_hdf5_roundtrip_matches_serial(tmp_path, parallel_hdf5_lane): + communicator = parallel_hdf5_lane if communicator is None: return rank = world_rank(communicator) @@ -237,10 +249,10 @@ def test_collective_hdf5_roundtrip_matches_serial(tmp_path): assert failure is None, failure -def test_collective_hdf5_refuses_rank_local_metadata_before_write(tmp_path): - communicator = _parallel_hdf5_world( - test_collective_hdf5_refuses_rank_local_metadata_before_write.__name__ - ) +def test_collective_hdf5_refuses_rank_local_metadata_before_write( + tmp_path, parallel_hdf5_lane, +): + communicator = parallel_hdf5_lane if communicator is None: return rank = world_rank(communicator) @@ -265,10 +277,10 @@ def test_collective_hdf5_refuses_rank_local_metadata_before_write(tmp_path): assert not tuple(shared_root.glob(".*must-not-exist*.tmp")) -def test_collective_hdf5_refuses_divergent_target_before_write(tmp_path): - communicator = _parallel_hdf5_world( - test_collective_hdf5_refuses_divergent_target_before_write.__name__ - ) +def test_collective_hdf5_refuses_divergent_target_before_write( + tmp_path, parallel_hdf5_lane, +): + communicator = parallel_hdf5_lane if communicator is None: return rank = world_rank(communicator) @@ -290,10 +302,10 @@ def test_collective_hdf5_refuses_divergent_target_before_write(tmp_path): assert not tuple(shared_root.glob("*must-not-exist.h5")) -def test_native_collective_hdf5_binding_failure_is_all_rank_consensus(tmp_path): - communicator = _parallel_hdf5_world( - test_native_collective_hdf5_binding_failure_is_all_rank_consensus.__name__ - ) +def test_native_collective_hdf5_binding_failure_is_all_rank_consensus( + tmp_path, parallel_hdf5_lane, +): + communicator = parallel_hdf5_lane if communicator is None: return rank = world_rank(communicator) diff --git a/tests/python/integration/mpi/test_scientific_output_mpi.py b/tests/python/integration/mpi/test_scientific_output_mpi.py index e01111287..189ce3610 100644 --- a/tests/python/integration/mpi/test_scientific_output_mpi.py +++ b/tests/python/integration/mpi/test_scientific_output_mpi.py @@ -128,6 +128,8 @@ def _shared_directory() -> Path: def _validate_native_binding_error_consensus(root: Path) -> None: """One malformed rank must fail before HDF5 while every peer receives the same cause.""" + lane = COMM.duplicate_observer_lane( + "scientific-output-mpi-hdf5-binding-validation") values = ( [[1.0, 2.0], [3.0, 4.0]] if RANK == 0 @@ -135,21 +137,24 @@ def _validate_native_binding_error_consensus(root: Path) -> None: ) error = None try: - _pops._write_parallel_hdf5( - COMM, - str(root / "binding-must-not-enter-hdf5.h5"), - "{}", - {"geometry/0000/coverage": np.zeros((2, 2), dtype=np.bool_)}, - ({ - "dataset": "fields/0000/values", - "dtype": np.dtype(np.float64).str, - "shape": (2, 2), - "pieces": ({"lower": (0, 0), "upper": (2, 2), "values": values},), - },), - ) - except RuntimeError as exc: - error = str(exc) - errors = allgather_value(COMM, error) + try: + _pops._write_parallel_hdf5( + lane, + str(root / "binding-must-not-enter-hdf5.h5"), + "{}", + {"geometry/0000/coverage": np.zeros((2, 2), dtype=np.bool_)}, + ({ + "dataset": "fields/0000/values", + "dtype": np.dtype(np.float64).str, + "shape": (2, 2), + "pieces": ({"lower": (0, 0), "upper": (2, 2), "values": values},), + },), + ) + except RuntimeError as exc: + error = str(exc) + errors = allgather_value(lane, error) + finally: + lane.close_collectively() if not all(item is not None and "binding input validation" in item for item in errors): raise AssertionError("rank-local binding fault did not reach all ranks: %r" % (errors,)) if len(set(errors)) != 1: From 836f2f2812e72a9bbb95de8c13c86d9a1bb02a44 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:17:31 +0200 Subject: [PATCH 069/109] docs(output): describe duplicated HDF5 observer lanes (ADC-683) --- .../SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md | 5 +++-- docs/design/exact-output-consumers.md | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md index e8bb1baf1..af5e0e05d 100644 --- a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md +++ b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md @@ -1491,8 +1491,9 @@ scientifiques choisissent obligatoirement un `ParallelMode` typé : d'un unique writer rang 0, `COLLECTIVE` pour les hyperslabs HDF5 MPIO exacts, ou `PER_RANK` pour des artefacts locaux qualifiés par rang et un reçu agrégé. Le mode, le format, la sélection, la cible et l'identité de chaque pièce native (`global_box_index`, `owner_rank`, `replicated`) sont authentifiés -entre rangs avant toute écriture. La route `COLLECTIVE` appelle le backend C++ HDF5 parallèle sur -`MPI_COMM_WORLD`; `h5py` reste uniquement un lecteur/écrivain série optionnel et n'est jamais un +entre rangs avant toute écriture. La route `COLLECTIVE` appelle le backend C++ HDF5 parallèle avec +la lane MPI dupliquée possédée par la session observateur ; le writer ne redécouvre ni n'emprunte +`MPI_COMM_WORLD`. `h5py` reste uniquement un lecteur/écrivain série optionnel et n'est jamais un transport MPI. Une dépendance HDF5 parallèle native absente, un mode incompatible ou un backend Kokkos GPU/device handle non supporté est refusé avant le constructeur de `System`/`AmrSystem`; aucune route série implicite ne remplace une demande MPI. diff --git a/docs/design/exact-output-consumers.md b/docs/design/exact-output-consumers.md index 8b81f518b..e2ab048b5 100644 --- a/docs/design/exact-output-consumers.md +++ b/docs/design/exact-output-consumers.md @@ -85,10 +85,11 @@ count, target suffix, or writer availability: gather, but only rank 0 prepares, verifies and atomically publishes the single-file writer. Preparation failures and the final receipt are broadcast to every participant. - `COLLECTIVE` requires a distributed context, an authenticated collective resource plan and the - native C++ parallel-HDF5 provider. Each rank writes only its exact non-overlapping native - hyperslabs with exactly one MPIO collective transfer per dataset and rank (including a select-none - transfer for a rank with no patch). A replicated AMR coarse patch is assigned to rank 0 for this - mode so it cannot overlap. + native C++ parallel-HDF5 provider. The observer runtime owns a duplicated MPI lane for the complete + writer session; neither the Python writer nor the native HDF5 adapter borrows or rediscovers the + process world. Each rank writes only its exact non-overlapping native hyperslabs with exactly one + MPIO collective transfer per dataset and rank (including a select-none transfer for a rank with no + patch). A replicated AMR coarse patch is assigned to rank 0 for this mode so it cannot overlap. - `PER_RANK` requires a distributed context and preserves each rank's exact local pieces, including explicitly replicated coarse pieces. Targets are rank-qualified before any file is opened. The transaction succeeds only after it aggregates one deterministic receipt per contiguous rank. From 5ee5cf795a71255e3c78272a06de8396bf54e6fc Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:38:52 +0200 Subject: [PATCH 070/109] refactor(output): isolate ROOT gathers on consumer lanes (ADC-683) --- include/pops/parallel/comm.hpp | 11 ++ include/pops/parallel/execution_lane.hpp | 108 +++++++++++++++--- include/pops/parallel/world_communicator.hpp | 11 -- include/pops/runtime/amr_system.hpp | 6 +- .../pops/runtime/output_piece_collective.hpp | 30 +++-- include/pops/runtime/system.hpp | 6 +- python/bindings/core/init/init_amr.cpp | 14 +-- python/bindings/core/init/init_system.cpp | 15 ++- python/pops/_pops.pyi | 8 +- python/pops/runtime/_runtime_consumers.py | 67 ++++++++++- src/runtime/amr/amr_system.cpp | 8 +- src/runtime/system/system_fields.cpp | 8 +- 12 files changed, 221 insertions(+), 71 deletions(-) diff --git a/include/pops/parallel/comm.hpp b/include/pops/parallel/comm.hpp index 02b6e8e06..66d009dbb 100644 --- a/include/pops/parallel/comm.hpp +++ b/include/pops/parallel/comm.hpp @@ -96,6 +96,17 @@ inline void require_mpi_success(int code, std::string_view operation) { throw_mpi_error(code, operation); } +inline int chunk_capacity(int ranks) { + const int divisor = std::max(1, ranks); + return std::max(1, std::numeric_limits::max() / divisor); +} + +inline const char* chunk_pointer(const std::string& payload, unsigned long long offset, int count) { + if (count == 0) + return nullptr; + return payload.data() + static_cast(offset); +} + inline bool comm_active_unlocked() noexcept { int initialized = 0; int finalized = 0; diff --git a/include/pops/parallel/execution_lane.hpp b/include/pops/parallel/execution_lane.hpp index b811d904e..eea796b0e 100644 --- a/include/pops/parallel/execution_lane.hpp +++ b/include/pops/parallel/execution_lane.hpp @@ -560,11 +560,20 @@ class ObserverMpiLane { throw std::out_of_range("observer collective root is outside the lane"); const int me = lane.rank(); - std::optional> result; + long length_overflow = 0; + if constexpr (sizeof(std::size_t) > sizeof(unsigned long long)) { + if (payload.size() > static_cast(std::numeric_limits::max())) + length_overflow = 1; + } + if (all_reduce_max(length_overflow, lane) != 0) + throw std::overflow_error("consumer gather payload exceeds the MPI length domain"); + const unsigned long long local_length = static_cast(payload.size()); + + std::vector lengths; long allocation_failed = 0; if (me == root) { try { - result.emplace(static_cast(ranks)); + lengths.resize(static_cast(ranks), 0ULL); } catch (const std::bad_alloc&) { allocation_failed = 1; } catch (const std::length_error&) { @@ -572,25 +581,94 @@ class ObserverMpiLane { } } if (all_reduce_max(allocation_failed, lane) != 0) - throw std::runtime_error("observer root could not allocate gathered results"); + throw std::runtime_error("consumer root could not allocate gathered lengths"); + detail::require_mpi_success( + MPI_Gather(&local_length, 1, MPI_UNSIGNED_LONG_LONG, me == root ? lengths.data() : nullptr, + 1, MPI_UNSIGNED_LONG_LONG, root, lane.native_handle()), + "MPI_Gather(consumer payload lengths)"); - for (int source = 0; source < ranks; ++source) { - std::string source_payload; - long copy_failed = 0; - if (me == source) { + unsigned long long maximum_length = local_length; + detail::require_mpi_success( + MPI_Allreduce(MPI_IN_PLACE, &maximum_length, 1, MPI_UNSIGNED_LONG_LONG, MPI_MAX, + lane.native_handle()), + "MPI_Allreduce(maximum consumer gather length)"); + + std::optional> result; + std::vector counts; + std::vector displacements; + allocation_failed = 0; + if (me == root) { + try { + result.emplace(static_cast(ranks)); + counts.resize(static_cast(ranks), 0); + displacements.resize(static_cast(ranks), 0); + for (int rank = 0; rank < ranks; ++rank) { + const unsigned long long length = lengths[static_cast(rank)]; + if (length > static_cast(std::numeric_limits::max())) { + allocation_failed = 1; + break; + } + (*result)[static_cast(rank)].resize(static_cast(length)); + } + } catch (const std::bad_alloc&) { + allocation_failed = 1; + } catch (const std::length_error&) { + allocation_failed = 1; + } + } + if (all_reduce_max(allocation_failed, lane) != 0) + throw std::runtime_error("consumer root could not allocate gathered payloads"); + + const int capacity = detail::chunk_capacity(ranks); + for (unsigned long long offset = 0; offset < maximum_length; + offset += static_cast(capacity)) { + int total = 0; + if (me == root) { + for (int rank = 0; rank < ranks; ++rank) { + const unsigned long long length = lengths[static_cast(rank)]; + const int count = offset < length + ? static_cast(std::min( + length - offset, static_cast(capacity))) + : 0; + counts[static_cast(rank)] = count; + displacements[static_cast(rank)] = total; + total += count; + } + } + std::vector round; + long round_allocation_failed = 0; + if (me == root) { try { - source_payload = payload; + round.resize(static_cast(total)); } catch (const std::bad_alloc&) { - copy_failed = 1; + round_allocation_failed = 1; } catch (const std::length_error&) { - copy_failed = 1; + round_allocation_failed = 1; } } - if (all_reduce_max(copy_failed, lane) != 0) - throw std::runtime_error("an observer rank could not stage its gather payload"); - std::string received = broadcast_bytes(std::move(source_payload), source); - if (me == root) - (*result)[static_cast(source)] = std::move(received); + if (all_reduce_max(round_allocation_failed, lane) != 0) + throw std::runtime_error("consumer root could not allocate a gathered chunk"); + const int send_count = + offset < local_length + ? static_cast(std::min( + local_length - offset, static_cast(capacity))) + : 0; + detail::require_mpi_success( + MPI_Gatherv(detail::chunk_pointer(payload, offset, send_count), send_count, MPI_BYTE, + me == root ? round.data() : nullptr, me == root ? counts.data() : nullptr, + me == root ? displacements.data() : nullptr, MPI_BYTE, root, + lane.native_handle()), + "MPI_Gatherv(consumer payload chunk)"); + if (me != root) + continue; + for (int rank = 0; rank < ranks; ++rank) { + const int count = counts[static_cast(rank)]; + if (count == 0) + continue; + std::copy_n( + round.data() + displacements[static_cast(rank)], count, + (*result)[static_cast(rank)].data() + static_cast(offset)); + } } return result; #else diff --git a/include/pops/parallel/world_communicator.hpp b/include/pops/parallel/world_communicator.hpp index 830936ff4..e66d2fb8d 100644 --- a/include/pops/parallel/world_communicator.hpp +++ b/include/pops/parallel/world_communicator.hpp @@ -60,17 +60,6 @@ inline int validated_collective_root(int root) { return root; } -inline int chunk_capacity(int ranks) { - const int divisor = std::max(1, ranks); - return std::max(1, std::numeric_limits::max() / divisor); -} - -inline const char* chunk_pointer(const std::string& payload, unsigned long long offset, int count) { - if (count == 0) - return nullptr; - return payload.data() + static_cast(offset); -} - #endif } // namespace detail diff --git a/include/pops/runtime/amr_system.hpp b/include/pops/runtime/amr_system.hpp index 7600c0c01..9d45216b1 100644 --- a/include/pops/runtime/amr_system.hpp +++ b/include/pops/runtime/amr_system.hpp @@ -58,7 +58,7 @@ namespace pops { -class WorldCommunicator; +class ObserverMpiLane; namespace runtime::program { class AmrProgramContext; } @@ -668,7 +668,7 @@ class AmrSystem { /// Exact rank-local valid-cell pieces for one qualified field provider. The returned metadata /// explicitly marks replicated level-zero ownership so output modes never infer it from box counts. std::vector output_field_local_pieces(const std::string& provider_slot, int level); - std::vector output_field_root_pieces(const WorldCommunicator& world, + std::vector output_field_root_pieces(const ObserverMpiLane& lane, const std::string& provider_slot, int level); /// Transaction bracket used by the accepted-state reader after complete payload preflight. Every /// hierarchy, @@ -1039,7 +1039,7 @@ class AmrSystem { /// without allocating a global level buffer. std::vector output_state_local_pieces(const std::string& name, int k); std::vector output_geometry_boxes(); - std::vector output_state_root_pieces(const WorldCommunicator& world, + std::vector output_state_root_pieces(const ObserverMpiLane& lane, const std::string& name, int k); /// Owner rank per box of level @p k (the shared layout's DistributionMapping), aligned with the /// level-@p k rows of patch_boxes(). The v3 checkpoint (ADC-542) serializes it so a restart diff --git a/include/pops/runtime/output_piece_collective.hpp b/include/pops/runtime/output_piece_collective.hpp index 258c52778..57b30bb4a 100644 --- a/include/pops/runtime/output_piece_collective.hpp +++ b/include/pops/runtime/output_piece_collective.hpp @@ -5,10 +5,10 @@ /// /// Local providers are evaluated on every rank under an all-rank error consensus. Metadata and /// IEEE-754 values are framed in a versioned, endian-stable native wire payload and transferred by -/// WorldCommunicator's chunked MPI_Gatherv transport. Only rank zero materializes the global piece -/// vector; Python never gathers NumPy arrays or executes an MPI collective. +/// an explicitly owned consumer lane. Only rank zero materializes the global piece vector; Python +/// never gathers NumPy arrays or executes an MPI collective. -#include +#include #include #include @@ -185,13 +185,21 @@ inline std::string current_exception_text() { /// Evaluate a local OutputPiece provider and gather its exact result onto MPI rank zero. template -std::vector output_pieces_to_root(const WorldCommunicator& world, +std::vector output_pieces_to_root(const ObserverMpiLane& lane, std::string operation_identity, Provider&& provider) { - world.require_active_mpi_world(); - const int rank = world.rank(); +#ifndef POPS_HAS_MPI + (void)lane; + (void)operation_identity; + (void)provider; + throw std::runtime_error("native output-piece ROOT gather requires an MPI-enabled build"); +#endif + if (!lane.active()) + throw std::runtime_error( + "native output-piece root gather requires an active consumer MPI lane"); + const int rank = lane.rank(); - const std::vector operations = world.allgather_bytes(operation_identity); + const std::vector operations = lane.allgather_bytes(operation_identity); if (!std::all_of(operations.begin(), operations.end(), [&](const std::string& value) { return value == operation_identity; })) throw std::invalid_argument("output-piece root gather arguments differ across MPI ranks"); @@ -215,19 +223,19 @@ std::vector output_pieces_to_root(const WorldCommunicator& world, local_error = detail::current_exception_text(); } - const std::vector errors = world.allgather_bytes(local_error); + const std::vector errors = lane.allgather_bytes(local_error); for (std::size_t source = 0; source < errors.size(); ++source) { if (!errors[source].empty()) throw std::runtime_error("native output-piece provider failed on rank " + std::to_string(source) + ": " + errors[source]); } - const std::optional> gathered = world.gather_bytes(packed, 0); + const std::optional> gathered = lane.gather_bytes(packed, 0); std::vector result; std::string root_error; if (rank == 0) { try { - if (!gathered || gathered->size() != static_cast(world.size())) + if (!gathered || gathered->size() != static_cast(lane.size())) throw std::runtime_error("native output-piece root gather has invalid rank cardinality"); for (std::size_t source = 0; source < gathered->size(); ++source) { std::vector decoded = @@ -248,7 +256,7 @@ std::vector output_pieces_to_root(const WorldCommunicator& world, root_error = detail::current_exception_text(); } } - root_error = world.broadcast_bytes(std::move(root_error), 0); + root_error = lane.broadcast_bytes(std::move(root_error), 0); if (!root_error.empty()) throw std::runtime_error("native output-piece reconstruction failed: " + root_error); return result; diff --git a/include/pops/runtime/system.hpp b/include/pops/runtime/system.hpp index 50bab6654..bf2b2288c 100644 --- a/include/pops/runtime/system.hpp +++ b/include/pops/runtime/system.hpp @@ -47,7 +47,7 @@ namespace pops { -class WorldCommunicator; +class ObserverMpiLane; class PreparedSystemLayoutTransfer; namespace component { @@ -1333,9 +1333,9 @@ class System { std::vector output_field_local_pieces(const std::string& provider_slot, int level); /// Collective ROOT views. Local provider errors are agreed before native MPI_Gatherv; only rank /// zero receives complete pieces and every non-root rank receives an empty vector. - std::vector output_state_root_pieces(const WorldCommunicator& world, + std::vector output_state_root_pieces(const ObserverMpiLane& lane, const std::string& name, int level) const; - std::vector output_field_root_pieces(const WorldCommunicator& world, + std::vector output_field_root_pieces(const ObserverMpiLane& lane, const std::string& provider_slot, int level); /// @} diff --git a/python/bindings/core/init/init_amr.cpp b/python/bindings/core/init/init_amr.cpp index c9a3b55be..dc38a4c83 100644 --- a/python/bindings/core/init/init_amr.cpp +++ b/python/bindings/core/init/init_amr.cpp @@ -1,5 +1,5 @@ #include "../bindings_detail.hpp" -#include +#include #include "boundary_component_install.hpp" #include "output_geometry_binding.hpp" @@ -956,16 +956,16 @@ void bind_amr_data(py::class_& cls) { "Exact compact valid-cell pieces of one qualified field owned by this rank.") .def( "output_field_root_pieces", - [](AmrSystem& s, const WorldCommunicator& world, const std::string& provider_slot, + [](AmrSystem& s, const ObserverMpiLane& lane, const std::string& provider_slot, int level) { std::vector pieces; { py::gil_scoped_release release; - pieces = s.output_field_root_pieces(world, provider_slot, level); + pieces = s.output_field_root_pieces(lane, provider_slot, level); } return output_pieces_to_python(pieces); }, - py::arg("world"), py::arg("provider_slot"), py::arg("level"), + py::arg("lane"), py::arg("provider_slot"), py::arg("level"), "Collectively gather compact field pieces in C++; complete only on MPI rank zero.") .def( "_output_geometry_snapshot", @@ -1008,15 +1008,15 @@ void bind_amr_data(py::class_& cls) { "Exact compact valid-cell pieces of one qualified state owned by this rank.") .def( "output_state_root_pieces", - [](AmrSystem& s, const WorldCommunicator& world, const std::string& name, int level) { + [](AmrSystem& s, const ObserverMpiLane& lane, const std::string& name, int level) { std::vector pieces; { py::gil_scoped_release release; - pieces = s.output_state_root_pieces(world, name, level); + pieces = s.output_state_root_pieces(lane, name, level); } return output_pieces_to_python(pieces); }, - py::arg("world"), py::arg("block"), py::arg("level"), + py::arg("lane"), py::arg("block"), py::arg("level"), "Collectively gather compact state pieces in C++; complete only on MPI rank zero.") .def( "set_block_level_state", diff --git a/python/bindings/core/init/init_system.cpp b/python/bindings/core/init/init_system.cpp index 40153d799..8e93c6ee9 100644 --- a/python/bindings/core/init/init_system.cpp +++ b/python/bindings/core/init/init_system.cpp @@ -1,5 +1,5 @@ #include "../bindings_detail.hpp" -#include +#include #include "boundary_component_install.hpp" #include "output_geometry_binding.hpp" @@ -921,28 +921,27 @@ void bind_system_data(py::class_& cls) { "Exact compact valid-cell field pieces owned by this rank.") .def( "output_state_root_pieces", - [](const System& s, const WorldCommunicator& world, const std::string& block, int level) { + [](const System& s, const ObserverMpiLane& lane, const std::string& block, int level) { std::vector pieces; { py::gil_scoped_release release; - pieces = s.output_state_root_pieces(world, block, level); + pieces = s.output_state_root_pieces(lane, block, level); } return output_pieces_to_python(pieces); }, - py::arg("world"), py::arg("block"), py::arg("level"), + py::arg("lane"), py::arg("block"), py::arg("level"), "Collectively gather compact state pieces in C++; complete only on MPI rank zero.") .def( "output_field_root_pieces", - [](System& s, const WorldCommunicator& world, const std::string& provider_slot, - int level) { + [](System& s, const ObserverMpiLane& lane, const std::string& provider_slot, int level) { std::vector pieces; { py::gil_scoped_release release; - pieces = s.output_field_root_pieces(world, provider_slot, level); + pieces = s.output_field_root_pieces(lane, provider_slot, level); } return output_pieces_to_python(pieces); }, - py::arg("world"), py::arg("provider_slot"), py::arg("level"), + py::arg("lane"), py::arg("provider_slot"), py::arg("level"), "Collectively gather compact field pieces in C++; complete only on MPI rank zero.") .def( "_output_geometry_snapshot", diff --git a/python/pops/_pops.pyi b/python/pops/_pops.pyi index 559803ef5..e9a3dfe32 100644 --- a/python/pops/_pops.pyi +++ b/python/pops/_pops.pyi @@ -283,10 +283,10 @@ class System: self, provider_slot: str, level: int ) -> tuple[dict[str, object], ...]: ... def output_state_root_pieces( - self, world: _NativeWorldCommunicator, block: str, level: int + self, lane: _NativeObserverMpiLane, block: str, level: int ) -> tuple[dict[str, object], ...]: ... def output_field_root_pieces( - self, world: _NativeWorldCommunicator, provider_slot: str, level: int + self, lane: _NativeObserverMpiLane, provider_slot: str, level: int ) -> tuple[dict[str, object], ...]: ... @@ -309,10 +309,10 @@ class AmrSystem: self, provider_slot: str, level: int ) -> tuple[dict[str, object], ...]: ... def output_state_root_pieces( - self, world: _NativeWorldCommunicator, block: str, level: int + self, lane: _NativeObserverMpiLane, block: str, level: int ) -> tuple[dict[str, object], ...]: ... def output_field_root_pieces( - self, world: _NativeWorldCommunicator, provider_slot: str, level: int + self, lane: _NativeObserverMpiLane, provider_slot: str, level: int ) -> tuple[dict[str, object], ...]: ... diff --git a/python/pops/runtime/_runtime_consumers.py b/python/pops/runtime/_runtime_consumers.py index 0d7dbddd1..70327ad31 100644 --- a/python/pops/runtime/_runtime_consumers.py +++ b/python/pops/runtime/_runtime_consumers.py @@ -1288,6 +1288,7 @@ def __init__(self, owner: Any) -> None: self._rank, self._size, self._communicator = rank, size, communicator self._observer_queues: dict[tuple[str, str], PostCommitObserverQueue] = {} self._observer_lanes: dict[tuple[str, str], Any] = {} + self._root_output_lanes: dict[str, Any] = {} self._observer_workers: dict[str, PostCommitObserverWorker] = {} self._observer_journals: dict[tuple[str, str], Any] = {} self._observer_preflight_sessions: dict[str, Any] = {} @@ -1321,6 +1322,14 @@ def __init__(self, owner: Any) -> None: ) self._builtin_catalyst_consumers = tuple(sorted(builtin_catalyst)) self._builtin_catalyst_run_started = False + self._root_output_consumers = tuple( + sorted( + candidate.qualified_id + for candidate in owner._consumer_graph.nodes + if candidate.kind is ConsumerKind.SCIENTIFIC_OUTPUT + and candidate.parallel_mode is ParallelMode.ROOT + ) + ) from pops import interfaces for manifest in owner._consumer_graph.nodes: @@ -1789,6 +1798,21 @@ def begin_post_commit_consumers(self, run_identity: Identity) -> None: """ self._observer_key("run-begin", run_identity) + if run_identity.token in self._closed_observer_runs: + raise RuntimeError("post-commit consumers cannot reopen an already closed run") + if self._root_output_consumers: + if run_identity.token in self._root_output_lanes: + raise RuntimeError( + "the ROOT scientific-output MPI lane is already active for this run" + ) + if self._communicator is None: + raise RuntimeError( + "ROOT scientific output lost its authenticated execution communicator" + ) + lane_identity = "scientific-output/root/%s" % run_identity.token + self._root_output_lanes[run_identity.token] = ( + self._communicator.duplicate_observer_lane(lane_identity) + ) if self._builtin_catalyst_consumers: if self._builtin_catalyst_run_started: raise RuntimeError( @@ -2215,6 +2239,23 @@ def flush_live_visualizations( self._observer_diagnostics.append(rendered) failures.append(rendered) if close: + root_lane = self._root_output_lanes.pop(run_identity.token, None) + if self._root_output_consumers and root_lane is None: + rendered = "ROOT scientific-output MPI lane disappeared before close" + if rendered not in self._observer_diagnostics: + self._observer_diagnostics.append(rendered) + failures.append(rendered) + elif root_lane is not None: + try: + root_lane.close_collectively() + except BaseException as error: + rendered = ( + "ROOT scientific-output MPI lane close failed: %s" + % _exception_text(error) + ) + if rendered not in self._observer_diagnostics: + self._observer_diagnostics.append(rendered) + failures.append(rendered) worker = self._observer_workers.pop(run_identity.token, None) if worker is not None: try: @@ -2255,6 +2296,20 @@ def close_live_visualizations( run_identity, close=True, raise_on_failure=raise_on_failure ) + def _root_output_communicator(self) -> Any: + """Return the one active duplicated lane used by native ROOT snapshot gathers.""" + + if not self._root_output_consumers: + raise RuntimeError("the ConsumerGraph declares no ROOT scientific output") + if len(self._root_output_lanes) != 1: + raise RuntimeError( + "ROOT scientific output requires exactly one active run-scoped MPI lane" + ) + lane = next(iter(self._root_output_lanes.values())) + if lane.active is not True or lane.closed is not False: + raise RuntimeError("ROOT scientific-output MPI lane is not active") + return lane + def diagnostic_restart_state(self) -> dict[str, Any]: """Return the complete last-accepted typed diagnostic registry.""" baselines = dict(self._baselines) @@ -3241,10 +3296,20 @@ def _distributed_pieces( else method_name ) try: + native_communicator = communicator + if mode is ParallelMode.ROOT: + lane_provider = getattr( + self._owner._publisher, "_root_output_communicator", None + ) + if not callable(lane_provider): + raise RuntimeError( + "ROOT scientific output has no run-scoped MPI lane provider" + ) + native_communicator = lane_provider() local = self._local_pieces( native_engine, selected_method, - (communicator, *args) if mode is ParallelMode.ROOT else args, + (native_communicator, *args) if mode is ParallelMode.ROOT else args, mode=mode, rank=rank, require_local_owner=mode is not ParallelMode.ROOT, diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index ae23d0cd3..6c28de408 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -1751,11 +1751,11 @@ std::vector AmrSystem::output_field_local_pieces(const std::string& return p_->runtime->output_field_local_pieces(provider_slot, level); } -std::vector AmrSystem::output_field_root_pieces(const WorldCommunicator& world, +std::vector AmrSystem::output_field_root_pieces(const ObserverMpiLane& lane, const std::string& provider_slot, int level) { return output_pieces_to_root( - world, detail::output_collective_identity("AmrSystem", "field", provider_slot, level), + lane, detail::output_collective_identity("AmrSystem", "field", provider_slot, level), [&] { return output_field_local_pieces(provider_slot, level); }); } @@ -4327,9 +4327,9 @@ std::vector AmrSystem::output_geometry_boxes() { return p_->runtime->output_geometry_boxes(); } -std::vector AmrSystem::output_state_root_pieces(const WorldCommunicator& world, +std::vector AmrSystem::output_state_root_pieces(const ObserverMpiLane& lane, const std::string& name, int k) { - return output_pieces_to_root(world, + return output_pieces_to_root(lane, detail::output_collective_identity("AmrSystem", "state", name, k), [&] { return output_state_local_pieces(name, k); }); } diff --git a/src/runtime/system/system_fields.cpp b/src/runtime/system/system_fields.cpp index 460c04b42..f6c9ff4ac 100644 --- a/src/runtime/system/system_fields.cpp +++ b/src/runtime/system/system_fields.cpp @@ -918,19 +918,19 @@ std::vector System::output_field_local_pieces(const std::string& pr return output_local_pieces(field, 0, false); } -std::vector System::output_state_root_pieces(const WorldCommunicator& world, +std::vector System::output_state_root_pieces(const ObserverMpiLane& lane, const std::string& name, int level) const { - return output_pieces_to_root(world, + return output_pieces_to_root(lane, detail::output_collective_identity("System", "state", name, level), [&] { return output_state_local_pieces(name, level); }); } -std::vector System::output_field_root_pieces(const WorldCommunicator& world, +std::vector System::output_field_root_pieces(const ObserverMpiLane& lane, const std::string& provider_slot, int level) { return output_pieces_to_root( - world, detail::output_collective_identity("System", "field", provider_slot, level), + lane, detail::output_collective_identity("System", "field", provider_slot, level), [&] { return output_field_local_pieces(provider_slot, level); }); } From 26b9b9643bdd7ac36bc8553acf2e2dd411c4754d Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:39:00 +0200 Subject: [PATCH 071/109] test(output): fence run-scoped ROOT MPI lanes (ADC-683) --- .../mpi/test_mpi_amr_distributed_coarse.cpp | 6 +- .../mpi/test_mpi_system_io_gather.cpp | 6 +- .../unit/parallel/test_world_communicator.cpp | 27 +++--- .../test_root_output_consumer_lane_fence.py | 46 ++++++++++ .../runtime/test_runtime_instance_gate.py | 87 +++++++++++++++++-- 5 files changed, 151 insertions(+), 21 deletions(-) create mode 100644 tests/python/architecture/test_root_output_consumer_lane_fence.py diff --git a/tests/cpp/integration/mpi/test_mpi_amr_distributed_coarse.cpp b/tests/cpp/integration/mpi/test_mpi_amr_distributed_coarse.cpp index 4af988742..978113157 100644 --- a/tests/cpp/integration/mpi/test_mpi_amr_distributed_coarse.cpp +++ b/tests/cpp/integration/mpi/test_mpi_amr_distributed_coarse.cpp @@ -40,6 +40,7 @@ #include "amr_tagging_test_authority.hpp" #include +#include #include #include @@ -302,7 +303,10 @@ static Result run(int n, int nsteps, double dt, bool distribute) { // contract of level_{state,potential}_global(0). R.state = sys.level_state_global(0); R.output_local_pieces = sys.output_state_local_pieces("gas", 0); - R.output_root_pieces = sys.output_state_root_pieces(WorldCommunicator::world(), "gas", 0); + auto output_lane = + ObserverMpiLane::duplicate_world_collectively("test/amr-distributed-coarse/root-output"); + R.output_root_pieces = sys.output_state_root_pieces(output_lane, "gas", 0); + output_lane.close_collectively(); R.phi = sys.potential(); R.phi_global = sys.level_potential_global(0); R.mass = sys.mass(); diff --git a/tests/cpp/integration/mpi/test_mpi_system_io_gather.cpp b/tests/cpp/integration/mpi/test_mpi_system_io_gather.cpp index 918ca7c7b..de7f4a1d8 100644 --- a/tests/cpp/integration/mpi/test_mpi_system_io_gather.cpp +++ b/tests/cpp/integration/mpi/test_mpi_system_io_gather.cpp @@ -53,6 +53,7 @@ #include #include +#include #include #include @@ -144,12 +145,12 @@ static int pops_run_test_mpi_system_io_gather(int argc, char** argv) { // === T1 : gather == reference connue (np-invariant), sur le champ fraichement pose =========== // Tous les rangs appellent les accesseurs collectifs ; le resultat egale BIT-A-BIT la reference. + auto output_lane = ObserverMpiLane::duplicate_world_collectively("test/system-io/root-output"); { const std::vector dG = sys.density_global("gas"); const std::vector sG = sys.state_global("gas"); const std::vector local = sys.output_state_local_pieces("gas", 0); - const std::vector root = - sys.output_state_root_pieces(WorldCommunicator::world(), "gas", 0); + const std::vector root = sys.output_state_root_pieces(output_lane, "gas", 0); chk(dG.size() == nn, "T1_density_global_size"); chk(sG.size() == 4 * nn, "T1_state_global_size"); chk(dG == rho_ref, "T1_density_global_eq_ref_no_double_count"); @@ -172,6 +173,7 @@ static int pops_run_test_mpi_system_io_gather(int argc, char** argv) { chk(piece.ncomp == 4 && piece.values == sG, "T1_output_state_root_values"); } } + output_lane.close_collectively(); // === T2 : apres des pas COLLECTIFS, gather == accesseur local sur le proprietaire ============ const double dt = 0.01; diff --git a/tests/cpp/unit/parallel/test_world_communicator.cpp b/tests/cpp/unit/parallel/test_world_communicator.cpp index 7c6028cc8..eca1caed8 100644 --- a/tests/cpp/unit/parallel/test_world_communicator.cpp +++ b/tests/cpp/unit/parallel/test_world_communicator.cpp @@ -111,12 +111,12 @@ TEST(WorldCommunicator, TransfersEmptyNullAndVariableSizedBytes) { } TEST(WorldCommunicator, GathersOutputPiecesOnlyOnRoot) { - pops::WorldCommunicator& world = pops::WorldCommunicator::world(); + auto lane = pops::ObserverMpiLane::duplicate_world_collectively("test/output-piece/gather"); #ifdef POPS_HAS_MPI - const int rank = world.rank(); - const int size = world.size(); + const int rank = lane.rank(); + const int size = lane.size(); std::vector result = pops::output_pieces_to_root( - world, pops::detail::output_collective_identity("test", "state", "tracer", 0), [rank] { + lane, pops::detail::output_collective_identity("test", "state", "tracer", 0), [rank] { pops::OutputPiece piece; piece.box = pops::PatchBox{0, rank, 0, rank, 0}; piece.global_box_index = rank; @@ -141,18 +141,19 @@ TEST(WorldCommunicator, GathersOutputPiecesOnlyOnRoot) { } #else EXPECT_THROW((void)pops::output_pieces_to_root( - world, pops::detail::output_collective_identity("test", "state", "tracer", 0), + lane, pops::detail::output_collective_identity("test", "state", "tracer", 0), [] { return std::vector{}; }), std::runtime_error); #endif + lane.close_collectively(); } TEST(WorldCommunicator, SelectsOneCanonicalReplicatedOutputContributor) { - pops::WorldCommunicator& world = pops::WorldCommunicator::world(); + auto lane = pops::ObserverMpiLane::duplicate_world_collectively("test/output-piece/replicated"); #ifdef POPS_HAS_MPI - const int rank = world.rank(); + const int rank = lane.rank(); std::vector result = pops::output_pieces_to_root( - world, pops::detail::output_collective_identity("test", "state", "replicated", 0), [rank] { + lane, pops::detail::output_collective_identity("test", "state", "replicated", 0), [rank] { pops::OutputPiece piece; piece.box = pops::PatchBox{0, 0, 0, 0, 0}; piece.global_box_index = 0; @@ -173,10 +174,10 @@ TEST(WorldCommunicator, SelectsOneCanonicalReplicatedOutputContributor) { EXPECT_TRUE(result.empty()); } #else - EXPECT_THROW( - (void)pops::output_pieces_to_root( - world, pops::detail::output_collective_identity("test", "state", "replicated", 0), - [] { return std::vector{}; }), - std::runtime_error); + EXPECT_THROW((void)pops::output_pieces_to_root( + lane, pops::detail::output_collective_identity("test", "state", "replicated", 0), + [] { return std::vector{}; }), + std::runtime_error); #endif + lane.close_collectively(); } diff --git a/tests/python/architecture/test_root_output_consumer_lane_fence.py b/tests/python/architecture/test_root_output_consumer_lane_fence.py new file mode 100644 index 000000000..e48dbe16f --- /dev/null +++ b/tests/python/architecture/test_root_output_consumer_lane_fence.py @@ -0,0 +1,46 @@ +"""ADC-683 fences for run-owned native ROOT scientific-output communication.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +COLLECTIVE = ROOT / "include/pops/runtime/output_piece_collective.hpp" +SYSTEM = ROOT / "include/pops/runtime/system.hpp" +AMR = ROOT / "include/pops/runtime/amr_system.hpp" +SYSTEM_BINDING = ROOT / "python/bindings/core/init/init_system.cpp" +AMR_BINDING = ROOT / "python/bindings/core/init/init_amr.cpp" +RUNTIME = ROOT / "python/pops/runtime/_runtime_consumers.py" +STUB = ROOT / "python/pops/_pops.pyi" + + +def test_native_root_output_surface_requires_an_owned_consumer_lane(): + collective = COLLECTIVE.read_text(encoding="utf-8") + system = SYSTEM.read_text(encoding="utf-8") + amr = AMR.read_text(encoding="utf-8") + + assert "WorldCommunicator" not in collective + assert "MPI_COMM_WORLD" not in collective + assert "const ObserverMpiLane& lane" in collective + assert "const ObserverMpiLane& lane" in system + assert "const ObserverMpiLane& lane" in amr + + +def test_python_root_output_bridge_rejects_the_process_world_type(): + system = SYSTEM_BINDING.read_text(encoding="utf-8") + amr = AMR_BINDING.read_text(encoding="utf-8") + stub = STUB.read_text(encoding="utf-8") + + assert "WorldCommunicator" not in system + assert "WorldCommunicator" not in amr + assert "const ObserverMpiLane& lane" in system + assert "const ObserverMpiLane& lane" in amr + assert "lane: _NativeObserverMpiLane" in stub + + +def test_runtime_materializes_and_closes_one_root_output_lane_per_run(): + runtime = RUNTIME.read_text(encoding="utf-8") + + assert 'lane_identity = "scientific-output/root/%s" % run_identity.token' in runtime + assert "self._communicator.duplicate_observer_lane(lane_identity)" in runtime + assert "root_lane.close_collectively()" in runtime + assert "native_communicator = lane_provider()" in runtime diff --git a/tests/python/unit/runtime/test_runtime_instance_gate.py b/tests/python/unit/runtime/test_runtime_instance_gate.py index 52941529b..7b549d975 100644 --- a/tests/python/unit/runtime/test_runtime_instance_gate.py +++ b/tests/python/unit/runtime/test_runtime_instance_gate.py @@ -259,13 +259,13 @@ def output_state_local_pieces(self, block, level): ) def output_state_root_pieces(self, communicator, block, level): - """Expose the exact singleton-world gather required by ROOT publication tests.""" - from pops._native_collectives import require_world, size + """Expose the exact duplicated consumer lane required by ROOT publication tests.""" + from pops._native_collectives import require_communicator, size expected = self._plan.execution_context.communicator - if communicator is not expected.handle: - raise ValueError("ROOT gather did not receive the installed communicator handle") - native = require_world(communicator) + native = require_communicator(communicator, allow_world=False) + if expected.identity != "MPI_COMM_WORLD": + raise ValueError("ROOT gather requires an MPI execution context") if size(native) != 1: raise RuntimeError( "runtime-instance unit executor only implements a singleton ROOT gather" @@ -1731,6 +1731,83 @@ def test_checkpoint_diagnostic_baseline_schema_is_finite_and_canonical(): ) +def test_root_output_lane_requires_one_active_run_scoped_communicator(): + from pops.runtime._runtime_consumers import RuntimeConsumerPublisher + + publisher = object.__new__(RuntimeConsumerPublisher) + lane = SimpleNamespace(active=True, closed=False) + publisher._root_output_consumers = ("scientific_output/root",) + publisher._root_output_lanes = {"run": lane} + assert publisher._root_output_communicator() is lane + + publisher._root_output_lanes = {} + with pytest.raises(RuntimeError, match="exactly one active"): + publisher._root_output_communicator() + + publisher._root_output_lanes = {"run": SimpleNamespace(active=False, closed=False)} + with pytest.raises(RuntimeError, match="not active"): + publisher._root_output_communicator() + + publisher._root_output_consumers = () + with pytest.raises(RuntimeError, match="declares no ROOT"): + publisher._root_output_communicator() + + +def test_root_output_lane_is_materialized_and_closed_once_per_run(): + from pops.runtime._runtime_consumers import RuntimeConsumerPublisher + + class _Lane: + active = True + closed = False + + def __init__(self): + self.close_calls = 0 + + def close_collectively(self): + self.close_calls += 1 + self.active = False + self.closed = True + + class _World: + def __init__(self, lane): + self.lane = lane + self.identities = [] + + def duplicate_observer_lane(self, identity): + self.identities.append(identity) + return self.lane + + run_identity = make_identity("run", {"case": "root-output-lane"}) + lane = _Lane() + world = _World(lane) + publisher = object.__new__(RuntimeConsumerPublisher) + publisher._root_output_consumers = ("scientific_output/root",) + publisher._root_output_lanes = {} + publisher._communicator = world + publisher._closed_observer_runs = set() + publisher._builtin_catalyst_consumers = () + publisher._builtin_catalyst_run_started = False + publisher._owner = SimpleNamespace( + _consumer_graph=SimpleNamespace(nodes=()), + ) + publisher._observer_diagnostics = [] + publisher._observer_workers = {} + publisher._observer_reports = {} + publisher._observer_queues = {} + publisher._observer_pending_failures = {} + + publisher.begin_post_commit_consumers(run_identity) + assert world.identities == ["scientific-output/root/%s" % run_identity.token] + assert publisher._root_output_communicator() is lane + + assert publisher.close_live_visualizations(run_identity) == () + assert lane.close_calls == 1 + assert publisher.close_live_visualizations(run_identity) == () + assert lane.close_calls == 1 + with pytest.raises(RuntimeError, match="already closed"): + publisher.begin_post_commit_consumers(run_identity) + + def test_diagnostic_component_requires_one_explicit_role_for_multicomponent_state(): from pops.runtime._runtime_consumers import RuntimeConsumerPublisher From fe1af95d3ea32fd5ca36f0e5c623ffbceb72e9ca Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:39:08 +0200 Subject: [PATCH 072/109] docs(output): document ROOT consumer lane ownership (ADC-683) --- ...TION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md | 12 +++++----- docs/design/exact-output-consumers.md | 22 +++++++++++-------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md index af5e0e05d..797cf5ff2 100644 --- a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md +++ b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md @@ -1491,11 +1491,13 @@ scientifiques choisissent obligatoirement un `ParallelMode` typé : d'un unique writer rang 0, `COLLECTIVE` pour les hyperslabs HDF5 MPIO exacts, ou `PER_RANK` pour des artefacts locaux qualifiés par rang et un reçu agrégé. Le mode, le format, la sélection, la cible et l'identité de chaque pièce native (`global_box_index`, `owner_rank`, `replicated`) sont authentifiés -entre rangs avant toute écriture. La route `COLLECTIVE` appelle le backend C++ HDF5 parallèle avec -la lane MPI dupliquée possédée par la session observateur ; le writer ne redécouvre ni n'emprunte -`MPI_COMM_WORLD`. `h5py` reste uniquement un lecteur/écrivain série optionnel et n'est jamais un -transport MPI. Une dépendance HDF5 parallèle native absente, un mode incompatible ou un backend -Kokkos GPU/device handle non supporté est refusé avant le +entre rangs avant toute écriture. La capture native `ROOT` reçoit uniquement une lane consommateur +dupliquée pour le run et la libère collectivement à sa fermeture ; les façades +`System`/`AmrSystem` n'acceptent plus le singleton monde pour cette route. La route `COLLECTIVE` +appelle le backend C++ HDF5 parallèle avec la lane MPI dupliquée possédée par la session observateur ; +le writer ne redécouvre ni n'emprunte `MPI_COMM_WORLD`. `h5py` reste uniquement un +lecteur/écrivain série optionnel et n'est jamais un transport MPI. Une dépendance HDF5 parallèle +native absente, un mode incompatible ou un backend Kokkos GPU/device handle non supporté est refusé avant le constructeur de `System`/`AmrSystem`; aucune route série implicite ne remplace une demande MPI. Les maillages non structurés, mobiles/déformables ou changeant de topologie, de nouvelles familles de diff --git a/docs/design/exact-output-consumers.md b/docs/design/exact-output-consumers.md index e2ab048b5..beac3137a 100644 --- a/docs/design/exact-output-consumers.md +++ b/docs/design/exact-output-consumers.md @@ -82,8 +82,10 @@ count, target suffix, or writer availability: - `SERIAL` requires the proved serial `ExecutionContext` (rank 0, size 1) and one complete snapshot. - `ROOT` requires a distributed context. Every rank participates in the authenticated native - gather, but only rank 0 prepares, verifies and atomically publishes the single-file writer. - Preparation failures and the final receipt are broadcast to every participant. + gather over a run-scoped duplicated consumer lane, but only rank 0 prepares, verifies and + atomically publishes the single-file writer. The native `System`/`AmrSystem` output bridge + accepts only that owned lane, never the process-world singleton. Preparation failures and the + final receipt are broadcast to every participant. - `COLLECTIVE` requires a distributed context, an authenticated collective resource plan and the native C++ parallel-HDF5 provider. The observer runtime owns a duplicated MPI lane for the complete writer session; neither the Python writer nor the native HDF5 adapter borrows or rediscovers the @@ -193,10 +195,11 @@ therefore write NPZ, HDF5 or the complete VTU/PVTU/PVD/state ParaView bundle. `q retained detached snapshots; a full queue deliberately applies backpressure. The selected format owns the topology. `SERIAL` uses the sole rank. `ROOT` performs the complete -snapshot gather on the main execution path, then writes from the rank-zero worker without worker -MPI. `PER_RANK` and `COLLECTIVE` run one worker per rank over a run-scoped communicator duplicated -collectively before any worker starts. That private lane has a distinct MPI context from -`MPI_COMM_WORLD`, so numerical and output collective orderings cannot alias. PoPS requires +snapshot gather on the main execution path over one run-scoped duplicated consumer lane, then +writes from the rank-zero worker without MPI. `PER_RANK` and `COLLECTIVE` run one worker per rank +over a run-scoped communicator duplicated collectively before any worker starts. Those private +lanes have distinct MPI contexts from `MPI_COMM_WORLD`, so numerical and output collective +orderings cannot alias. PoPS requires `MPI_THREAD_MULTIPLE`, authenticates the lane on every worker call and fixes distributed `max_attempts` to one: retrying after entry into an MPI publication would not be safe. Supported mode combinations remain those of the format itself; in particular, ParaView has no `COLLECTIVE` mode and @@ -385,9 +388,10 @@ re-emission; a rank-local `KeyboardInterrupt`/`SystemExit` cannot split collecti - HDF5 uses native datasets and `read_hdf5()` verification. Serial/root fields must be complete. Collective mode requires the compiled C++ parallel-HDF5 route before preparation; every rank writes its declared non-overlapping hyperslabs through the exact authenticated communicator and - the manifest authenticates all pieces. A synchronous consumer uses the execution communicator; - an asynchronous consumer uses its private duplicated worker lane. Python never emulates this - mode with a gather-to-root writer: the compiled provider owns the MPIO dataset transfers. + the manifest authenticates all pieces. The HDF5 session uses its private duplicated observer lane; + neither synchronous nor asynchronous publication borrows the process world. Python never + emulates this mode with a gather-to-root writer: the compiled provider owns the MPIO dataset + transfers. Partition validation scales with piece count rather than global cell count, and shared geometry is written once by rank zero. Unlike the default relayed PVTU topology, the single collective HDF5 target is opened by every rank through parallel HDF5/MPI-IO and must therefore be genuinely From a34649cc5fb471f8370cea4d14083a8179afd204 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:50:22 +0200 Subject: [PATCH 073/109] feat(numerics): bind generated flux provider slots --- include/pops/numerics/fv/flux_interfaces.hpp | 80 +++++++++++++++++++- 1 file changed, 76 insertions(+), 4 deletions(-) diff --git a/include/pops/numerics/fv/flux_interfaces.hpp b/include/pops/numerics/fv/flux_interfaces.hpp index a8e33b31d..937f46dd5 100644 --- a/include/pops/numerics/fv/flux_interfaces.hpp +++ b/include/pops/numerics/fv/flux_interfaces.hpp @@ -10,9 +10,11 @@ #include #include +#include #include #include #include +#include namespace pops { @@ -127,6 +129,46 @@ inline constexpr int flux_provider_count = [] { return kAuxBaseComps; }(); +template +inline constexpr bool has_qualified_flux_provider_requirements = requires { + Model::n_flux_providers; + Model::flux_provider_requirements; +}; + +/// Authenticate the generated logical provider ABI before a device pack can be instantiated. +/// +/// Hand-written C++ test models may omit both members. Generated models must provide both, and +/// every selected provider must be available, fully qualified, and backed by one in-range native +/// storage slot. The binder consumes exactly these rows; they are not inspection-only metadata. +template +consteval bool qualified_flux_provider_requirements_valid() { + constexpr bool has_count = requires { Model::n_flux_providers; }; + constexpr bool has_rows = requires { Model::flux_provider_requirements; }; + if constexpr (has_count != has_rows) { + return false; + } else if constexpr (!has_count) { + return true; + } else { + if (Model::n_flux_providers < 0 || static_cast(Model::n_flux_providers) != + Model::flux_provider_requirements.size()) + return false; + const auto nonempty = [](const char* value) { return value != nullptr && value[0] != '\0'; }; + for (std::size_t index = 0; index < Model::flux_provider_requirements.size(); ++index) { + const auto& row = Model::flux_provider_requirements[index]; + if (!row.available || row.storage_slot < 0 || + row.storage_slot >= flux_provider_count || !nonempty(row.owner_qid) || + !nonempty(row.space_kind) || !nonempty(row.space_name) || !nonempty(row.component) || + !nonempty(row.representation) || !nonempty(row.centering) || !nonempty(row.layout) || + !nonempty(row.producer)) + return false; + for (std::size_t previous = 0; previous < index; ++previous) + if (Model::flux_provider_requirements[previous].storage_slot == row.storage_slot) + return false; + } + return true; + } +} + /// Exact, model-qualified values before they are sealed into a bound device pack. /// /// Unlike the historical global Aux object this type has exactly the width requested by Model. @@ -136,6 +178,8 @@ inline constexpr int flux_provider_count = [] { template struct FluxProviderValues { static constexpr int size = flux_provider_count; + static_assert(qualified_flux_provider_requirements_valid(), + "generated physical flux provider requirements are invalid"); static_assert(size >= kAuxBaseComps, "physical flux provider packs must declare the required base providers"); static_assert(size <= kAuxMaxComps, @@ -176,15 +220,43 @@ POPS_HD BoundFluxProviders bind_flux_providers(const FluxProviderValues(values); } +namespace detail { + +template +inline constexpr int qualified_flux_provider_storage_slot = + Model::flux_provider_requirements[Index].storage_slot; + +template +POPS_HD BoundFluxProviders bind_qualified_flux_providers_at( + const Storage& storage, int i, int j, std::index_sequence) { + FluxProviderValues values{}; + ((values[qualified_flux_provider_storage_slot] = + storage(i, j, qualified_flux_provider_storage_slot)), + ...); + return bind_flux_providers(values); +} + +} // namespace detail + /// Bind one exact provider pack directly from native field storage. The caller supplies a /// model-qualified component count at compile time; there is no global Aux object, truncation, or /// zero-on-missing branch on this path. template POPS_HD BoundFluxProviders bind_flux_providers_at(const Storage& storage, int i, int j) { - FluxProviderValues values{}; - for (int component = 0; component < FluxProviderValues::size; ++component) - values[component] = storage(i, j, component); - return bind_flux_providers(values); + if constexpr (has_qualified_flux_provider_requirements) { + static_assert(qualified_flux_provider_requirements_valid(), + "generated physical flux provider requirements are invalid"); + constexpr std::size_t count = qualified_flux_provider_requirements_valid() + ? static_cast(Model::n_flux_providers) + : 0; + return detail::bind_qualified_flux_providers_at(storage, i, j, + std::make_index_sequence{}); + } else { + FluxProviderValues values{}; + for (int component = 0; component < FluxProviderValues::size; ++component) + values[component] = storage(i, j, component); + return bind_flux_providers(values); + } } template From 2cc1f15d4388a96c62a77b4f26c4b462a3984bf5 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:50:26 +0200 Subject: [PATCH 074/109] tests: prove qualified native flux binding --- .../unit/numerics/test_flux_interfaces.cpp | 68 +++++++++++++++++++ .../test_flux_interface_fences.py | 8 +++ .../codegen/test_compiler_model_provider.py | 2 + 3 files changed, 78 insertions(+) diff --git a/tests/cpp/unit/numerics/test_flux_interfaces.cpp b/tests/cpp/unit/numerics/test_flux_interfaces.cpp index 7c3d7a211..c8f5efa9f 100644 --- a/tests/cpp/unit/numerics/test_flux_interfaces.cpp +++ b/tests/cpp/unit/numerics/test_flux_interfaces.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -68,6 +69,49 @@ struct ProviderStorage { } }; +struct QualifiedProviderAdvect : ProviderAdvect { + static constexpr int n_flux_providers = 1; + inline static constexpr std::array + flux_provider_requirements{{ + {"model::qualified", "field", "electric", "grad_x", "scalar", "cell", "", + "layout::primary", "", "field::electric", true, 1}, + }}; +}; + +struct UnavailableQualifiedProviderAdvect : ProviderAdvect { + static constexpr int n_flux_providers = 1; + inline static constexpr std::array + flux_provider_requirements{{ + {"model::unavailable", "field", "electric", "grad_x", "scalar", "cell", "", + "layout::primary", "", "field::electric", false, 1}, + }}; +}; + +struct IncompleteQualifiedProviderAdvect : ProviderAdvect { + static constexpr int n_flux_providers = 1; +}; + +struct DuplicateQualifiedProviderAdvect : ProviderAdvect { + static constexpr int n_flux_providers = 2; + inline static constexpr std::array + flux_provider_requirements{{ + {"model::duplicate", "field", "electric", "grad_x", "scalar", "cell", "", + "layout::primary", "", "field::electric", true, 1}, + {"model::duplicate", "field", "magnetic", "grad_x", "scalar", "cell", "", + "layout::primary", "", "field::magnetic", true, 1}, + }}; +}; + +struct CountingProviderStorage { + pops::Real values[3]{pops::Real(11), pops::Real(4), pops::Real(13)}; + mutable int reads[3]{}; + + POPS_HD pops::Real operator()(int, int, int component) const { + ++reads[component]; + return values[component]; + } +}; + template auto providers(std::initializer_list values = {}) { pops::FluxProviderValues resolved{}; @@ -253,6 +297,30 @@ TEST(test_flux_interfaces, provider_pack_is_model_qualified_and_failure_action_i pops::TransactionFailureAction::kAbortRun); } +TEST(test_flux_interfaces, generated_provider_requirements_own_native_slot_reads) { + static_assert(pops::has_qualified_flux_provider_requirements); + static_assert(pops::qualified_flux_provider_requirements_valid()); + static_assert( + !pops::qualified_flux_provider_requirements_valid()); + static_assert( + !pops::qualified_flux_provider_requirements_valid()); + static_assert( + !pops::qualified_flux_provider_requirements_valid()); + + const CountingProviderStorage storage{}; + const auto bound = pops::bind_flux_providers_at(storage, 0, 0); + EXPECT_EQ(storage.reads[0], 0); + EXPECT_EQ(storage.reads[1], 1); + EXPECT_EQ(storage.reads[2], 0); + + const QualifiedProviderAdvect::State state{pops::Real(3)}; + const auto trace = pops::make_face_trace(state, bound); + const auto density = + pops::PhysicalFluxView{QualifiedProviderAdvect{}}.evaluate( + trace, pops::FaceContext::axis_aligned(0)); + EXPECT_DOUBLE_EQ(density.value[0], pops::Real(12)); +} + TEST(test_flux_interfaces, failed_evaluation_never_publishes_a_density) { const Advect physical{}; const Advect::State state{pops::Real(3)}; diff --git a/tests/python/architecture/test_flux_interface_fences.py b/tests/python/architecture/test_flux_interface_fences.py index 1083f28e5..03efb02b7 100644 --- a/tests/python/architecture/test_flux_interface_fences.py +++ b/tests/python/architecture/test_flux_interface_fences.py @@ -48,6 +48,14 @@ def test_bound_native_flux_pack_is_exact_and_does_not_store_global_aux(): assert "FluxDensity checked_density() const" in header +def test_generated_flux_pack_metadata_controls_native_storage_reads(): + header = _behavior(ROOT / "include/pops/numerics/fv/flux_interfaces.hpp") + assert "qualified_flux_provider_requirements_valid" in header + assert "qualified_flux_provider_storage_slot" in header + assert "std::make_index_sequence" in header + assert "generated physical flux provider requirements are invalid" in header + + def test_provider_selection_is_qualified_and_never_returns_a_neutral_value(): source = (ROOT / "python/pops/model/provider_pack.py").read_text(encoding="utf-8") assert "def select(" in source diff --git a/tests/python/unit/codegen/test_compiler_model_provider.py b/tests/python/unit/codegen/test_compiler_model_provider.py index 28f6e0aef..c915bb37c 100644 --- a/tests/python/unit/codegen/test_compiler_model_provider.py +++ b/tests/python/unit/codegen/test_compiler_model_provider.py @@ -132,6 +132,8 @@ def test_facade_and_formula_carrier_share_one_minimal_flux_provider_pack(): assert rows[0]["key"]["owner_qid"] in source assert '"grad_x"' in source assert "true, 1" in source + assert "static constexpr int n_flux_providers = 1;" in source + assert "flux_provider_requirements" in source def test_field_dependent_flux_without_provider_fails_before_native_source(): From 3c33ea6948a07b852b62ba955640de214b283770 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:50:32 +0200 Subject: [PATCH 075/109] docs: record generated flux provider ABI --- CHANGELOG.md | 3 +++ docs/ARCHITECTURE.md | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89338adb0..b9849c6ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning ### Changed +- Generated physical-flux bricks now make their qualified provider requirements executable native + ABI evidence: the binder validates every row at compile time and reads only its declared storage + slots instead of scanning the model's complete auxiliary width. - AMR checkpoint capability reports now distinguish same-rank bit-identical replay from non-bit-identical rank-count rematerialization with Dense persisted histories. The M3 gate executes the persisted two-rank to one-rank restart proof. The explicit `RegridOnRestart()` diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 1da4cb10d..a9dfc20fb 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -767,6 +767,11 @@ model-qualified `FaceTrace` values plus `FaceContext` and returns a typed densit `SpatialOperator` alone applies face and cell measures. Provider packs are selected from exact `(owner, space kind, space name, component)` identities. Missing, unavailable or contract-mismatched providers fail during selection; homonymous components from different owners never alias. +Generated physical models carry those qualified rows as `flux_provider_requirements`. The native +binder validates their count, qualification, availability, unique in-range storage slots and then +loads only those declared slots into the model-qualified device pack. Hand-written C++ test models +that do not declare this generated ABI retain the full-width fixture path; generated PoPS models +never use that route. ## Limitations From df46de301fbb10b9b048a293d874e53feddc06d8 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:51:31 +0200 Subject: [PATCH 076/109] release: prove installed-wheel component packages --- scripts/final_release_contract.py | 4 +++ scripts/release_preflight.py | 52 ++++++++++++++++++++++++++++++- scripts/run_final_gate.py | 35 +++++++++++++++++++-- 3 files changed, 88 insertions(+), 3 deletions(-) diff --git a/scripts/final_release_contract.py b/scripts/final_release_contract.py index 39658bcde..85c7e7b8d 100644 --- a/scripts/final_release_contract.py +++ b/scripts/final_release_contract.py @@ -35,6 +35,10 @@ # The published wheel matrix is CPU/Kokkos Serial without MPI or parallel HDF5. The full suite still # runs; this supported-platform subset is repeated with a strict all-pass/no-hidden-skip policy. PYTHON_REQUIRED_SELECTION = "not mpi and not hdf5" +INSTALLED_COMPONENT_PACKAGE_NODEID = ( + "tests/python/integration/native_loader/test_external_component_package.py" + "::test_source_component_executes_through_generic_native_loader_and_flux_consumer" +) REQUIRED_RELEASE_GATES = ( "official_build", "installed_wheel", diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index 57820f96c..6b4993dde 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -21,6 +21,7 @@ from final_release_contract import ( FINAL_EXAMPLES, + INSTALLED_COMPONENT_PACKAGE_NODEID, PYTHON_REQUIRED_SELECTION, REQUIRED_PROOF_MARKERS, REQUIRED_RELEASE_GATES, @@ -485,7 +486,7 @@ def _evidence(path: Path, contract: Any, commit: str, runtime: dict[str, str]) - for name in ("native_conformance", "python_conformance"): evidence = gates[name]["evidence"] expected = {"required_lane"} if name == "native_conformance" \ - else {"required_lane", "selection"} + else {"required_lane", "selection", "installed_component_package"} if not isinstance(evidence, dict) or set(evidence) != expected: raise PreflightError("release evidence %s lane is malformed" % name) lane = evidence["required_lane"] @@ -502,6 +503,55 @@ def _evidence(path: Path, contract: Any, commit: str, runtime: dict[str, str]) - label="%s JUnit" % name) if gates["python_conformance"]["evidence"]["selection"] != PYTHON_REQUIRED_SELECTION: raise PreflightError("release evidence Python required-lane selection drifted") + component = gates["python_conformance"]["evidence"]["installed_component_package"] + if not isinstance(component, dict) or set(component) != {"nodeid", "headers", "lane"}: + raise PreflightError("release evidence installed component package lane is malformed") + if component["nodeid"] != INSTALLED_COMPONENT_PACKAGE_NODEID \ + or component["headers"] != "installed-wheel": + raise PreflightError("release evidence installed component package authority drifted") + lane = component["lane"] + if not isinstance(lane, dict) or set(lane) != { + "path", "sha256", "tests", "failures", "skips_or_xfails"}: + raise PreflightError("release evidence installed component package JUnit is malformed") + if lane["tests"] != 1 or lane["failures"] != 0 or lane["skips_or_xfails"] != 0: + raise PreflightError("release evidence installed component package lane is not all-pass") + component_report = Path(lane["path"]).resolve() + if not _inside(directory, component_report): + raise PreflightError( + "release evidence installed component package JUnit path escapes its directory") + _artifact_file( + directory, + component_report.relative_to(directory), + lane["sha256"], + label="installed component package JUnit", + ) + component_commands = [ + command for command in gates["python_conformance"]["commands"] + if INSTALLED_COMPONENT_PACKAGE_NODEID in command["argv"] + ] + if len(component_commands) != 1: + raise PreflightError( + "release evidence must execute the installed component package node exactly once") + component_argv = component_commands[0]["argv"] + include_assignments = [ + argument for argument in component_argv if argument.startswith("POPS_INCLUDE=") + ] + if include_assignments != ["POPS_INCLUDE="] \ + or "POPS_PROVE_INSTALLED_COMPONENT_PACKAGE=1" not in component_argv: + raise PreflightError( + "installed component package proof must use only wheel-owned headers") + expected_suffix = [ + "python", + "-m", + "pytest", + "-q", + "-s", + INSTALLED_COMPONENT_PACKAGE_NODEID, + "--junitxml", + lane["path"], + ] + if component_argv[-len(expected_suffix):] != expected_suffix: + raise PreflightError("installed component package proof command drifted") _examples_evidence(directory, gates, runtime) diff --git a/scripts/run_final_gate.py b/scripts/run_final_gate.py index 0fbbbf2bc..61de40157 100644 --- a/scripts/run_final_gate.py +++ b/scripts/run_final_gate.py @@ -29,6 +29,7 @@ from final_release_contract import ( FINAL_EXAMPLES, FINAL_SPECIFICATION, + INSTALLED_COMPONENT_PACKAGE_NODEID, PYTHON_REQUIRED_SELECTION, REQUIRED_PROOF_MARKERS, REQUIRED_RELEASE_GATES, @@ -93,7 +94,11 @@ def _outside_checkout(path: Path) -> Path: raise FinalGateError("--evidence must be outside the checkout: %s" % resolved) -def _conda_command(arguments: Sequence[str]) -> list[str]: +def _conda_command( + arguments: Sequence[str], + *, + pops_include: Path | None = ROOT / "include", +) -> list[str]: """Run inside the same conda installation selected by the gate process. A login shell is deliberately forbidden here: user startup files may rewrite ``PATH`` and @@ -142,7 +147,7 @@ def _conda_command(arguments: Sequence[str]) -> list[str]: "PYTHONPATH=", "PYTHONNOUSERSITE=1", "POPS_REQUIRE_NATIVE_TESTS=1", - "POPS_INCLUDE=" + str((ROOT / "include").resolve()), + "POPS_INCLUDE=" + ("" if pops_include is None else str(pops_include.resolve())), *arguments, ] @@ -558,9 +563,35 @@ def main(argv: Sequence[str] | None = None) -> int: "--junitxml", str(python_junit), ])) _require_no_hidden_skip(required_stdout) + installed_component_junit = ( + evidence_root / "reports" / "installed-component-package.xml" + ) + installed_component_stdout = recorder.run( + "python_conformance", + _conda_command( + [ + "POPS_PROVE_INSTALLED_COMPONENT_PACKAGE=1", + "python", + "-m", + "pytest", + "-q", + "-s", + INSTALLED_COMPONENT_PACKAGE_NODEID, + "--junitxml", + str(installed_component_junit), + ], + pops_include=None, + ), + ) + _require_no_hidden_skip(installed_component_stdout) recorder.rows["python_conformance"]["evidence"] = { "required_lane": _junit_summary(python_junit), "selection": PYTHON_REQUIRED_SELECTION, + "installed_component_package": { + "nodeid": INSTALLED_COMPONENT_PACKAGE_NODEID, + "headers": "installed-wheel", + "lane": _junit_summary(installed_component_junit), + }, } signed_runtime_sha256 = _signed_runtime_sha256( recorder.rows["codesign"]["evidence"]) From 90354cfec1e2f86b73d047c40f4514592e6407ba Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:53:35 +0200 Subject: [PATCH 077/109] release: isolate component package evidence audit --- scripts/release_preflight.py | 105 +++++++++++++++++++---------------- 1 file changed, 56 insertions(+), 49 deletions(-) diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index 6b4993dde..c4771bf84 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -438,6 +438,61 @@ def _examples_evidence( raise PreflightError("release evidence restart proof markers drifted for %s" % key) +def _installed_component_package_evidence( + directory: Path, + python_conformance: dict[str, Any], +) -> None: + component = python_conformance["evidence"]["installed_component_package"] + if not isinstance(component, dict) or set(component) != {"nodeid", "headers", "lane"}: + raise PreflightError("release evidence installed component package lane is malformed") + if component["nodeid"] != INSTALLED_COMPONENT_PACKAGE_NODEID \ + or component["headers"] != "installed-wheel": + raise PreflightError("release evidence installed component package authority drifted") + lane = component["lane"] + if not isinstance(lane, dict) or set(lane) != { + "path", "sha256", "tests", "failures", "skips_or_xfails"}: + raise PreflightError("release evidence installed component package JUnit is malformed") + if lane["tests"] != 1 or lane["failures"] != 0 or lane["skips_or_xfails"] != 0: + raise PreflightError("release evidence installed component package lane is not all-pass") + component_report = Path(lane["path"]).resolve() + if not _inside(directory, component_report): + raise PreflightError( + "release evidence installed component package JUnit path escapes its directory") + _artifact_file( + directory, + component_report.relative_to(directory), + lane["sha256"], + label="installed component package JUnit", + ) + component_commands = [ + command for command in python_conformance["commands"] + if INSTALLED_COMPONENT_PACKAGE_NODEID in command["argv"] + ] + if len(component_commands) != 1: + raise PreflightError( + "release evidence must execute the installed component package node exactly once") + component_argv = component_commands[0]["argv"] + include_assignments = [ + argument for argument in component_argv if argument.startswith("POPS_INCLUDE=") + ] + if include_assignments != ["POPS_INCLUDE="] \ + or "POPS_PROVE_INSTALLED_COMPONENT_PACKAGE=1" not in component_argv: + raise PreflightError( + "installed component package proof must use only wheel-owned headers") + expected_suffix = [ + "python", + "-m", + "pytest", + "-q", + "-s", + INSTALLED_COMPONENT_PACKAGE_NODEID, + "--junitxml", + lane["path"], + ] + if component_argv[-len(expected_suffix):] != expected_suffix: + raise PreflightError("installed component package proof command drifted") + + def _evidence(path: Path, contract: Any, commit: str, runtime: dict[str, str]) -> None: payload = json.loads(path.read_text(encoding="utf-8")) expected = {"schema_version", "producer", "commit_sha", "package_version", "contract_sha256", @@ -503,55 +558,7 @@ def _evidence(path: Path, contract: Any, commit: str, runtime: dict[str, str]) - label="%s JUnit" % name) if gates["python_conformance"]["evidence"]["selection"] != PYTHON_REQUIRED_SELECTION: raise PreflightError("release evidence Python required-lane selection drifted") - component = gates["python_conformance"]["evidence"]["installed_component_package"] - if not isinstance(component, dict) or set(component) != {"nodeid", "headers", "lane"}: - raise PreflightError("release evidence installed component package lane is malformed") - if component["nodeid"] != INSTALLED_COMPONENT_PACKAGE_NODEID \ - or component["headers"] != "installed-wheel": - raise PreflightError("release evidence installed component package authority drifted") - lane = component["lane"] - if not isinstance(lane, dict) or set(lane) != { - "path", "sha256", "tests", "failures", "skips_or_xfails"}: - raise PreflightError("release evidence installed component package JUnit is malformed") - if lane["tests"] != 1 or lane["failures"] != 0 or lane["skips_or_xfails"] != 0: - raise PreflightError("release evidence installed component package lane is not all-pass") - component_report = Path(lane["path"]).resolve() - if not _inside(directory, component_report): - raise PreflightError( - "release evidence installed component package JUnit path escapes its directory") - _artifact_file( - directory, - component_report.relative_to(directory), - lane["sha256"], - label="installed component package JUnit", - ) - component_commands = [ - command for command in gates["python_conformance"]["commands"] - if INSTALLED_COMPONENT_PACKAGE_NODEID in command["argv"] - ] - if len(component_commands) != 1: - raise PreflightError( - "release evidence must execute the installed component package node exactly once") - component_argv = component_commands[0]["argv"] - include_assignments = [ - argument for argument in component_argv if argument.startswith("POPS_INCLUDE=") - ] - if include_assignments != ["POPS_INCLUDE="] \ - or "POPS_PROVE_INSTALLED_COMPONENT_PACKAGE=1" not in component_argv: - raise PreflightError( - "installed component package proof must use only wheel-owned headers") - expected_suffix = [ - "python", - "-m", - "pytest", - "-q", - "-s", - INSTALLED_COMPONENT_PACKAGE_NODEID, - "--junitxml", - lane["path"], - ] - if component_argv[-len(expected_suffix):] != expected_suffix: - raise PreflightError("installed component package proof command drifted") + _installed_component_package_evidence(directory, gates["python_conformance"]) _examples_evidence(directory, gates, runtime) From 9642bacf8327e4a16bb7d2dd52b22eea6537c54a Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:54:57 +0200 Subject: [PATCH 078/109] test(release): lock wheel-owned AOT evidence --- .../architecture/test_final_release_gate.py | 105 ++++++++++++++++++ .../test_external_component_package.py | 26 ++++- 2 files changed, 130 insertions(+), 1 deletion(-) diff --git a/tests/python/architecture/test_final_release_gate.py b/tests/python/architecture/test_final_release_gate.py index 4f5909843..513661e48 100644 --- a/tests/python/architecture/test_final_release_gate.py +++ b/tests/python/architecture/test_final_release_gate.py @@ -1,6 +1,7 @@ """Source-only contract checks for the final release gate (ADC-695).""" from __future__ import annotations +import copy import hashlib import importlib.util import json @@ -138,6 +139,110 @@ def test_final_gate_pins_one_conda_environment_and_native_headers( assert "bash" not in command +def test_installed_component_lane_clears_checkout_headers(monkeypatch, tmp_path): + executable = tmp_path / "conda" + executable.write_text("#!/bin/sh\nexit 0\n") + executable.chmod(0o755) + monkeypatch.setenv("POPS_CONDA_EXE", str(executable)) + command = gate._conda_command( + [ + "POPS_PROVE_INSTALLED_COMPONENT_PACKAGE=1", + "python", + "-m", + "pytest", + contract.INSTALLED_COMPONENT_PACKAGE_NODEID, + ], + pops_include=None, + ) + + assert [ + argument for argument in command if argument.startswith("POPS_INCLUDE=") + ] == ["POPS_INCLUDE="] + assert "POPS_PROVE_INSTALLED_COMPONENT_PACKAGE=1" in command + assert str((ROOT / "include").resolve()) not in command + + +def test_installed_component_node_is_real_and_rejects_mock_native_routes(): + relative, node = contract.INSTALLED_COMPONENT_PACKAGE_NODEID.split("::", 1) + source = (ROOT / relative).read_text(encoding="utf-8") + assert "def %s(" % node in source + helper = source.split("def _require_installed_component_package_proof()", 1)[1].split( + "\ndef ", 1 + )[0] + assert "Path(_pops.__file__).resolve()" in helper + assert "importlib.machinery.EXTENSION_SUFFIXES" in helper + assert "_pops.__has_kokkos__ is True" in helper + assert '["schema_version"] == 1' in helper + test_body = source.split("def %s(" % node, 1)[1].split("\ndef ", 1)[0] + assert test_body.index("_require_installed_component_package_proof()") \ + < test_body.index("compile_component(component)") + + +def test_preflight_authenticates_exact_installed_component_lane(tmp_path): + report = tmp_path / "reports" / "installed-component-package.xml" + report.parent.mkdir() + report.write_text( + '', + encoding="utf-8", + ) + lane = { + "path": str(report), + "sha256": hashlib.sha256(report.read_bytes()).hexdigest(), + "tests": 1, + "failures": 0, + "skips_or_xfails": 0, + } + argv = [ + "/proof/conda", + "run", + "--no-capture-output", + "-n", + "pops", + "/usr/bin/env", + "PYTHONPATH=", + "PYTHONNOUSERSITE=1", + "POPS_REQUIRE_NATIVE_TESTS=1", + "POPS_INCLUDE=", + "POPS_PROVE_INSTALLED_COMPONENT_PACKAGE=1", + "python", + "-m", + "pytest", + "-q", + "-s", + contract.INSTALLED_COMPONENT_PACKAGE_NODEID, + "--junitxml", + str(report), + ] + row = { + "commands": [{"argv": argv}], + "evidence": { + "installed_component_package": { + "nodeid": contract.INSTALLED_COMPONENT_PACKAGE_NODEID, + "headers": "installed-wheel", + "lane": lane, + }, + }, + } + preflight._installed_component_package_evidence(tmp_path, row) + + source_headers = copy.deepcopy(row) + source_headers["commands"][0]["argv"][ + source_headers["commands"][0]["argv"].index("POPS_INCLUDE=") + ] = "POPS_INCLUDE=/checkout/include" + with pytest.raises(preflight.PreflightError, match="wheel-owned headers"): + preflight._installed_component_package_evidence(tmp_path, source_headers) + + skipped = copy.deepcopy(row) + skipped["evidence"]["installed_component_package"]["lane"]["skips_or_xfails"] = 1 + with pytest.raises(preflight.PreflightError, match="not all-pass"): + preflight._installed_component_package_evidence(tmp_path, skipped) + + duplicate = copy.deepcopy(row) + duplicate["commands"].append(copy.deepcopy(duplicate["commands"][0])) + with pytest.raises(preflight.PreflightError, match="exactly once"): + preflight._installed_component_package_evidence(tmp_path, duplicate) + + def test_final_gate_honours_explicit_conda_executable(monkeypatch, tmp_path): executable = tmp_path / "conda" executable.write_text("#!/bin/sh\nexit 0\n") diff --git a/tests/python/integration/native_loader/test_external_component_package.py b/tests/python/integration/native_loader/test_external_component_package.py index 1e6e79b57..65a94aeff 100644 --- a/tests/python/integration/native_loader/test_external_component_package.py +++ b/tests/python/integration/native_loader/test_external_component_package.py @@ -1,8 +1,10 @@ """Collected native package test: compile, audit, install, load and call the real ABI consumer.""" from __future__ import annotations -import json +import importlib.machinery import importlib.util +import json +import os import subprocess import sys from dataclasses import replace @@ -38,6 +40,27 @@ EXAMPLE = ROOT / "examples/final/EXEMPLE_SPEC_FINALE_ADVECTION_SCALAIRE_COMPLET.py" +def _require_installed_component_package_proof() -> None: + if os.environ.get("POPS_PROVE_INSTALLED_COMPONENT_PACKAGE") != "1": + return + package_root = Path(pops.__file__).resolve().parent + wheel_include = (package_root / "include").resolve() + assert wheel_include.is_dir() + assert (wheel_include / "pops_headers.manifest").is_file() + assert Path(pops_include()).resolve() == wheel_include + + from pops import _pops + + native_path = Path(_pops.__file__).resolve() + assert native_path.parent == package_root + assert any( + native_path.name.endswith(suffix) + for suffix in importlib.machinery.EXTENSION_SUFFIXES + ) + assert _pops.__has_kokkos__ is True + assert _pops.__native_loader_contract__["schema_version"] == 1 + + def _manifest(*, generic: bool = True, device: str = "cpu") -> ComponentManifest: interface = interfaces.NumericalFlux return ComponentManifest( @@ -352,6 +375,7 @@ def _writer_source(manifest: ComponentManifest) -> bytes: def test_source_component_executes_through_generic_native_loader_and_flux_consumer(tmp_path): + _require_installed_component_package_proof() manifest = _manifest() source = _source(manifest) (tmp_path / "average.cpp").write_bytes(source) From 9266ecff76f20adec67a695dc086fd6d81881c7c Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:55:51 +0200 Subject: [PATCH 079/109] docs: record installed-wheel AOT proof --- CHANGELOG.md | 4 ++++ docs/design/external-component-packages.md | 8 +++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89338adb0..6b8919463 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,10 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning ### Changed +- The final release gate now proves an external source component against the exact installed wheel: + its isolated AOT lane clears the checkout-owned `POPS_INCLUDE`, requires the wheel-owned signed + header tree and native Kokkos extension, compiles/installs/loads the component, and retains one + exact no-skip/no-xfail JUnit result whose node ID and command are reauthenticated by preflight. - AMR checkpoint capability reports now distinguish same-rank bit-identical replay from non-bit-identical rank-count rematerialization with Dense persisted histories. The M3 gate executes the persisted two-rank to one-rank restart proof. The explicit `RegridOnRestart()` diff --git a/docs/design/external-component-packages.md b/docs/design/external-component-packages.md index caed1db16..7ddcfc464 100644 --- a/docs/design/external-component-packages.md +++ b/docs/design/external-component-packages.md @@ -88,4 +88,10 @@ artifacts on the declared failure path. Other devices, scalar types and dimensions remain unavailable until a target variant and every interface operation prove them. The wheel ships the exact signed PoPS header tree under -`pops/include`, so AOT compilation does not depend on a source checkout. +`pops/include`, so AOT compilation does not depend on a source checkout. The release gate proves +this independently of the ordinary source conformance lane: it clears `POPS_INCLUDE`, imports the +retained installed wheel with an empty `PYTHONPATH`, requires `pops_include()` to resolve exactly to +that wheel's `pops/include`, and rejects a stub or mocked native route before compiling, installing, +loading and invoking the external numerical-flux component. The one exact pytest node produces an +all-pass JUnit report; release preflight reauthenticates its node ID, command, wheel-header authority +and report digest, and refuses skips, xfails, duplicate execution or a checkout header override. From 9a853c5e420d021a9711a937a7b001af4e946759 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 07:56:38 +0200 Subject: [PATCH 080/109] release: normalize retained JUnit paths --- scripts/release_preflight.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index c4771bf84..285cc2dae 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -460,7 +460,7 @@ def _installed_component_package_evidence( "release evidence installed component package JUnit path escapes its directory") _artifact_file( directory, - component_report.relative_to(directory), + component_report.relative_to(directory).as_posix(), lane["sha256"], label="installed component package JUnit", ) @@ -554,7 +554,7 @@ def _evidence(path: Path, contract: Any, commit: str, runtime: dict[str, str]) - report = Path(lane["path"]).resolve() if not _inside(directory, report): raise PreflightError("release evidence %s JUnit path escapes its directory" % name) - _artifact_file(directory, report.relative_to(directory), lane["sha256"], + _artifact_file(directory, report.relative_to(directory).as_posix(), lane["sha256"], label="%s JUnit" % name) if gates["python_conformance"]["evidence"]["selection"] != PYTHON_REQUIRED_SELECTION: raise PreflightError("release evidence Python required-lane selection drifted") From 9ad7aba00a64e37aed2da135d9bb1f667e085f39 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:02:43 +0200 Subject: [PATCH 081/109] feat(runtime): capture due projection balance evidence (ADC-686) --- .../runtime/program/amr_program_context.hpp | 44 ++++++++++++++++++- .../pops/runtime/program/program_context.hpp | 24 ++++++++++ .../program/program_execution_services.hpp | 32 +++++++++++++- .../runtime/program/program_runtime_state.hpp | 33 +++++++++++--- python/pops/codegen/program_balance_due.py | 7 +++ src/runtime/amr/amr_system.cpp | 3 ++ src/runtime/system/system_impl.hpp | 3 ++ 7 files changed, 137 insertions(+), 9 deletions(-) diff --git a/include/pops/runtime/program/amr_program_context.hpp b/include/pops/runtime/program/amr_program_context.hpp index 05127b51c..ac1fa24fc 100644 --- a/include/pops/runtime/program/amr_program_context.hpp +++ b/include/pops/runtime/program/amr_program_context.hpp @@ -36,6 +36,7 @@ #include #include #include // AmrRuntime (the engine the driver wraps) +#include #include #include // GridContext (per-level Schur assembly seam, ADC-633) #include // AmrSystem (the facade: params / block map / engine) @@ -3164,6 +3165,45 @@ class AmrProgramContext : public ProgramExecutionServices { void program_execution_apply_projection_(int runtime_block, MultiFab& state) const { eng_->project_level_state(static_cast(runtime_block), level_, state); } + std::optional> program_execution_projection_balance_integrals_( + int program_block, const MultiFab& state) const { + const std::size_t runtime_block = static_cast(sys_block(program_block)); + if (level_ < 0 || level_ >= nlev()) + throw std::out_of_range("AMR Program projection balance active level is out of range"); + const MultiFab& live = eng_->level_state(runtime_block, level_); + if (state.box_array().boxes() != live.box_array().boxes() || + state.dmap().ranks() != live.dmap().ranks() || state.ncomp() != live.ncomp() || + state.n_grow() != live.n_grow() || state.local_size() != live.local_size()) + throw std::invalid_argument( + "AMR Program projection balance candidate changed its exact level layout"); + + std::vector views; + views.reserve(static_cast(nlev())); + for (int level = 0; level < nlev(); ++level) { + const Geometry geometry = eng_->level_geom(level); + const MultiFab* values = level == level_ ? &state : &eng_->level_state(runtime_block, level); + views.push_back({values, geometry.dx(), geometry.dy()}); + } + const int next = level_ + 1 < nlev() ? level_ + 1 : -1; + MultiFab mask = pops::runtime::amr::composite_detail::active_mask(views, level_, next); + std::vector result(static_cast(state.ncomp()), 0.0); + for (int component = 0; component < state.ncomp(); ++component) + result[static_cast(component)] = + static_cast(pops::runtime::amr::composite_detail::local_sum( + state, mask, component, pops::runtime::amr::composite_detail::CompositeSumKind::Sum)); + if (!eng_->level_is_replicated(level_)) + all_reduce_sum_inplace(result.data(), result.size()); + const Geometry geometry = eng_->level_geom(level_); + const double cell_measure = + static_cast(geometry.dx()) * static_cast(geometry.dy()); + if (!std::isfinite(cell_measure) || cell_measure <= 0.0) + throw std::runtime_error( + "AMR Program projection balance requires a positive finite cell measure"); + std::vector integrated(result.size(), Real(0)); + for (std::size_t component = 0; component < result.size(); ++component) + integrated[component] = static_cast(cell_measure * result[component]); + return integrated; + } Real program_execution_hmin_() const { return eng_->level_hmin(level_); } Real program_execution_max_wave_speed_(int runtime_block, const MultiFab& state) const { return eng_->level_max_speed(static_cast(runtime_block), level_, state); @@ -3273,8 +3313,8 @@ class AmrProgramContext : public ProgramExecutionServices { const HistoryRegistration& registration) const { return pops::detail::AmrHistoryOps::initialized(*eng_, registration.name); } - double program_execution_history_slot_dt_storage_( - const HistoryRegistration& registration, int lag) const { + double program_execution_history_slot_dt_storage_(const HistoryRegistration& registration, + int lag) const { return pops::detail::AmrHistoryOps::slot_dt(*eng_, registration.name, lag); } void program_execution_set_history_initialized_storage_(const HistoryRegistration& registration, diff --git a/include/pops/runtime/program/program_context.hpp b/include/pops/runtime/program/program_context.hpp index d4780ec9d..fab5a6518 100644 --- a/include/pops/runtime/program/program_context.hpp +++ b/include/pops/runtime/program/program_context.hpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -660,6 +661,29 @@ class ProgramContext : public ProgramExecutionServices { void program_execution_apply_projection_(int runtime_block, MultiFab& state) const { sys_->block_project(runtime_block, state); } + std::optional> program_execution_projection_balance_integrals_( + int program_block, const MultiFab& state) const { + // The public polar diagnostic path has no exact per-cell volume provider yet. Keep automatic + // evidence absent instead of relabelling Cartesian dx*dy as a polar measure; authored balance + // terms remain available and the future selector must fail closed on this missing producer. + if (sys_->program_is_polar()) + return std::nullopt; + const GridContext context = program_execution_block_grid_context_(program_block); + const Real cell_measure = context.geom.dx() * context.geom.dy(); + if (!std::isfinite(static_cast(cell_measure)) || cell_measure <= Real(0)) + throw std::runtime_error( + "Uniform Program projection balance requires a positive finite cell measure"); + RelativeCellMeasure measure; + if (context.domain_mask != nullptr) { + measure.active_cells = context.domain_mask; + measure.inverse_volume_fraction = context.eb_inverse_volume_fraction; + } + std::vector result(static_cast(state.ncomp()), Real(0)); + for (int component = 0; component < state.ncomp(); ++component) + result[static_cast(component)] = + cell_measure * pops::reduce_sum(state, component, measure); + return result; + } Real program_execution_hmin_() const { return sys_->cfl_min_dx(); } Real program_execution_max_wave_speed_(int runtime_block, const MultiFab& state) const { return sys_->block_max_speed(runtime_block, state); diff --git a/include/pops/runtime/program/program_execution_services.hpp b/include/pops/runtime/program/program_execution_services.hpp index 93bd66963..e7d1943a1 100644 --- a/include/pops/runtime/program/program_execution_services.hpp +++ b/include/pops/runtime/program/program_execution_services.hpp @@ -502,9 +502,33 @@ class ProgramExecutionServices { /// Project one candidate state through the exact authored block closure. /// /// Program-to-runtime block qualification is topology-independent. The provider owns only the - /// Uniform or level-qualified native projection call. + /// Uniform or level-qualified native projection call. When a generated Balance route is due, the + /// provider also supplies exact metric-integrated component values before and after projection; + /// their signed delta stays qualified by runtime block/level/component in the attempt mailbox. void apply_projection(int block, MultiFab& state) const { - provider_().program_execution_apply_projection_(sys_block(block), state); + const int runtime_block = sys_block(block); + ProgramRuntimeState& runtime = program_runtime_state_(); + if (!runtime.automatic_balance_capture_due()) { + provider_().program_execution_apply_projection_(runtime_block, state); + return; + } + const std::optional> before = + provider_().program_execution_projection_balance_integrals_(block, state); + provider_().program_execution_apply_projection_(runtime_block, state); + if (!before) + return; + const std::optional> after = + provider_().program_execution_projection_balance_integrals_(block, state); + if (!after || before->size() != after->size() || + before->size() != static_cast(state.ncomp())) + throw std::runtime_error( + "Program projection balance provider changed its conservative component width"); + const int level = program_resource_field_level(); + for (int component = 0; component < state.ncomp(); ++component) + runtime.record_automatic_balance_term(runtime_block, level, component, "projection", + (*after)[static_cast(component)] - + (*before)[static_cast(component)], + "ProgramExecutionServices"); } /// Minimum physical cell size used by the native CFL authority. @@ -1397,6 +1421,10 @@ class ProgramExecutionServices { provider_().program_execution_record_balance_term_(route, term, value); } + void note_automatic_balance_capture_due(bool due) const { + program_runtime_state_().note_automatic_balance_capture_due(due, "ProgramExecutionServices"); + } + void note_step_projection(const std::string& name) const { program_runtime_state_().note_step_projection(name); } diff --git a/include/pops/runtime/program/program_runtime_state.hpp b/include/pops/runtime/program/program_runtime_state.hpp index e27984c07..406faaeed 100644 --- a/include/pops/runtime/program/program_runtime_state.hpp +++ b/include/pops/runtime/program/program_runtime_state.hpp @@ -300,6 +300,11 @@ struct ProgramRuntimeState { /// by accepted_balance_terms(). The owning facade snapshots this map with the rest of the attempt, /// so rejection cannot leak automatic evidence into a retry. std::map automatic_balance_terms_; + /// Monotone attempt-local decision emitted by generated code before any Program operator runs. + /// It is the OR of the exact ConsumerGraph-derived route decisions for this public step. Keeping + /// this separate from step_balance_terms_ lets projection operators execute before their later + /// Program.record_balance sinks without losing due automatic evidence. + bool automatic_balance_due_ = false; /// Attempt-local outer accepted-step target used by ConsumerGraph-fused balance guards. Program /// substeps temporarily publish their window-start macro step through the facade, so generated /// balance code must not infer the public target from `macro_step()+1`. @@ -852,13 +857,30 @@ struct ProgramRuntimeState { entry->second += value; } - /// Whether a compiled Program has actually emitted a due Balance route in this attempt. + /// Whether generated code proved that at least one Balance route is due in this attempt. /// - /// Generated balance records are cadence-guarded before their reductions. Reflux executes after - /// the Program body, so observing a non-empty authored mailbox here avoids every extra native - /// reduction on an off-cadence or replay step without introducing a second scheduler. + /// The exact ConsumerGraph-derived decision is emitted before any Program operator, so both an + /// in-body projection and post-body reflux observe the same cadence without a second scheduler. [[nodiscard]] bool automatic_balance_capture_due() const noexcept { - return !balance_replay_active_ && !step_balance_terms_.empty(); + return !balance_replay_active_ && automatic_balance_due_; + } + + /// Publish one generated ConsumerGraph due decision before Program operators execute. + /// + /// Several compiled Program invocations may share one outer accepted-step window. The marker is + /// therefore monotone inside an attempt and is reset only at attempt entry. Static-false routes + /// emit no call, so a run without Balance consumers retains no generated hot-path branch. + void note_automatic_balance_capture_due(bool due, const std::string& runtime) { + if (balance_replay_active_) { + if (due) + throw std::logic_error(runtime + + "::note_automatic_balance_capture_due cannot enable replay capture"); + return; + } + if (!balance_due_window_active_) + throw std::logic_error( + runtime + "::note_automatic_balance_capture_due requires an active public-step window"); + automatic_balance_due_ = automatic_balance_due_ || due; } /// Accumulate one signed, metric-integrated native operator contribution. @@ -904,6 +926,7 @@ struct ProgramRuntimeState { step_projections_.clear(); step_balance_terms_.clear(); automatic_balance_terms_.clear(); + automatic_balance_due_ = false; balance_due_window_active_ = false; balance_due_target_step_ = 0; balance_step_completed_ = false; diff --git a/python/pops/codegen/program_balance_due.py b/python/pops/codegen/program_balance_due.py index 59c323b8a..1a57dafe5 100644 --- a/python/pops/codegen/program_balance_due.py +++ b/python/pops/codegen/program_balance_due.py @@ -212,6 +212,7 @@ def emit_balance_due_guards( if type(lowering) is not BalanceDueLowering: raise TypeError("balance due guard emission requires BalanceDueLowering") contract = json.dumps(lowering.contract.token) + automatic_tokens = [] for index, (route, periods) in enumerate(sorted(lowering.route_periods.items())): if not periods: token = "false" @@ -223,7 +224,13 @@ def emit_balance_due_guards( ] token = "balance_due_%d" % index lines.append("const bool %s = (%s);" % (token, " || ".join(calls))) + automatic_tokens.append(token) var[("balance_due_route", route)] = token + if automatic_tokens: + lines.append( + "ctx.note_automatic_balance_capture_due(%s);" + % (" || ".join(automatic_tokens)) + ) var[("balance_guarded_values",)] = lowering.guarded_values var[("balance_record_routes",)] = lowering.record_routes diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index cf5d1d318..3e0403973 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -463,6 +463,7 @@ struct AmrSystem::Impl { std::map program_diagnostics; std::map step_balance_terms; std::map automatic_balance_terms; + bool automatic_balance_due = false; bool balance_step_completed = false; bool balance_program_was_due = false; pops::runtime::program::CacheManager cache; @@ -511,6 +512,7 @@ struct AmrSystem::Impl { copy_value_map_into(program_diagnostics, impl.program_.diagnostics_); copy_value_map_into(step_balance_terms, impl.program_.step_balance_terms_); copy_value_map_into(automatic_balance_terms, impl.program_.automatic_balance_terms_); + automatic_balance_due = impl.program_.automatic_balance_due_; balance_step_completed = impl.program_.balance_step_completed_; balance_program_was_due = impl.program_.balance_program_was_due_; // AMR currently owns its native cache/history rings inside AmrRuntime. These two shared @@ -545,6 +547,7 @@ struct AmrSystem::Impl { copy_value_map_into(impl.program_.diagnostics_, program_diagnostics); copy_value_map_into(impl.program_.step_balance_terms_, step_balance_terms); copy_value_map_into(impl.program_.automatic_balance_terms_, automatic_balance_terms); + impl.program_.automatic_balance_due_ = automatic_balance_due; impl.program_.balance_step_completed_ = balance_step_completed; impl.program_.balance_program_was_due_ = balance_program_was_due; impl.program_.cache_ = cache; diff --git a/src/runtime/system/system_impl.hpp b/src/runtime/system/system_impl.hpp index 40fddf113..dfcde1e1c 100644 --- a/src/runtime/system/system_impl.hpp +++ b/src/runtime/system/system_impl.hpp @@ -618,6 +618,7 @@ struct System::Impl { std::map program_diagnostics; std::map step_balance_terms; std::map automatic_balance_terms; + bool automatic_balance_due; bool balance_step_completed; bool balance_program_was_due; pops::runtime::program::CacheManager cache; @@ -645,6 +646,7 @@ struct System::Impl { program_diagnostics(impl.program_.diagnostics_), step_balance_terms(impl.program_.step_balance_terms_), automatic_balance_terms(impl.program_.automatic_balance_terms_), + automatic_balance_due(impl.program_.automatic_balance_due_), balance_step_completed(impl.program_.balance_step_completed_), balance_program_was_due(impl.program_.balance_program_was_due_), cache(impl.program_.cache_), @@ -680,6 +682,7 @@ struct System::Impl { impl.program_.diagnostics_ = program_diagnostics; impl.program_.step_balance_terms_ = step_balance_terms; impl.program_.automatic_balance_terms_ = automatic_balance_terms; + impl.program_.automatic_balance_due_ = automatic_balance_due; impl.program_.balance_step_completed_ = balance_step_completed; impl.program_.balance_program_was_due_ = balance_program_was_due; impl.program_.cache_ = cache; From ab41987f9d52ac88f4d83ef8e9e303fb21647a9a Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:02:57 +0200 Subject: [PATCH 082/109] test(balance): fence projection evidence cadence (ADC-686) --- .../runtime/test_program_runtime.cpp | 25 ++++ ...test_automatic_projection_balance_fence.py | 115 ++++++++++++++++++ .../python/unit/time/test_time_ops_polish.py | 11 +- 3 files changed, 148 insertions(+), 3 deletions(-) create mode 100644 tests/python/architecture/test_automatic_projection_balance_fence.py diff --git a/tests/cpp/integration/runtime/test_program_runtime.cpp b/tests/cpp/integration/runtime/test_program_runtime.cpp index f36e55e23..7dbdd6202 100644 --- a/tests/cpp/integration/runtime/test_program_runtime.cpp +++ b/tests/cpp/integration/runtime/test_program_runtime.cpp @@ -139,6 +139,31 @@ TEST(ProgramRuntime, BalanceDueWindowUsesTheOuterAcceptedStepAndCleansUpOnFailur EXPECT_THROW((void)state.balance_consumer_is_due(contract, route, 4, "test"), std::logic_error); } +TEST(ProgramRuntime, AutomaticBalanceDueMarkerIsAttemptLocalMonotoneAndReplaySafe) { + runtime::program::ProgramRuntimeState state; + + EXPECT_FALSE(state.automatic_balance_capture_due()); + EXPECT_THROW(state.note_automatic_balance_capture_due(true, "test"), std::logic_error); + state.run_balance_due_window(0, "test", [&] { + state.note_automatic_balance_capture_due(false, "test"); + EXPECT_FALSE(state.automatic_balance_capture_due()); + state.note_automatic_balance_capture_due(true, "test"); + EXPECT_TRUE(state.automatic_balance_capture_due()); + state.note_automatic_balance_capture_due(false, "test"); + EXPECT_TRUE(state.automatic_balance_capture_due()); + }); + EXPECT_TRUE(state.automatic_balance_capture_due()); + + state.begin_step_projection_report(); + EXPECT_FALSE(state.automatic_balance_capture_due()); + state.run_balance_replay("test", [&] { + state.note_automatic_balance_capture_due(false, "test"); + EXPECT_FALSE(state.automatic_balance_capture_due()); + EXPECT_THROW(state.note_automatic_balance_capture_due(true, "test"), std::logic_error); + }); + EXPECT_FALSE(state.automatic_balance_capture_due()); +} + TEST(ProgramRuntime, SelectiveReplayCompilesBalanceOffAndRestoresTheGuard) { runtime::program::ProgramRuntimeState state; const std::string contract = "pops.balance-due-contract.v1:sha256:" + std::string(64, '3'); diff --git a/tests/python/architecture/test_automatic_projection_balance_fence.py b/tests/python/architecture/test_automatic_projection_balance_fence.py new file mode 100644 index 000000000..9c0e6e5fd --- /dev/null +++ b/tests/python/architecture/test_automatic_projection_balance_fence.py @@ -0,0 +1,115 @@ +"""ADC-686: projection balance evidence is due-only, metric, and still private.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +PROGRAM_STATE = ROOT / "include" / "pops" / "runtime" / "program" / "program_runtime_state.hpp" +EXECUTION_SERVICES = ( + ROOT / "include" / "pops" / "runtime" / "program" / "program_execution_services.hpp" +) +UNIFORM_CONTEXT = ROOT / "include" / "pops" / "runtime" / "program" / "program_context.hpp" +AMR_CONTEXT = ROOT / "include" / "pops" / "runtime" / "program" / "amr_program_context.hpp" +BALANCE_CODEGEN = ROOT / "python" / "pops" / "codegen" / "program_balance_due.py" +UNIFORM_IMPL = ROOT / "src" / "runtime" / "system" / "system_impl.hpp" +AMR_IMPL = ROOT / "src" / "runtime" / "amr" / "amr_system.cpp" + + +def _between(text: str, begin: str, end: str) -> str: + return text.split(begin, 1)[1].split(end, 1)[0] + + +def test_generated_due_marker_precedes_operators_and_is_attempt_local() -> None: + codegen = BALANCE_CODEGEN.read_text() + emit = _between( + codegen, + "def emit_balance_due_guards(", + "\ndef balance_value_due_expression(", + ) + assert "automatic_tokens = []" in emit + assert "if automatic_tokens:" in emit + assert "ctx.note_automatic_balance_capture_due(%s);" in emit + + state = PROGRAM_STATE.read_text() + assert "bool automatic_balance_due_ = false;" in state + capture_due = _between( + state, + "[[nodiscard]] bool automatic_balance_capture_due() const noexcept", + "/// Accumulate one signed, metric-integrated native operator contribution.", + ) + assert "!balance_replay_active_ && automatic_balance_due_" in capture_due + assert "automatic_balance_due_ = automatic_balance_due_ || due;" in capture_due + + attempt_entry = _between( + state, + "void begin_step_projection_report()", + "void note_step_projection(", + ) + assert "automatic_balance_due_ = false;" in attempt_entry + + uniform = UNIFORM_IMPL.read_text() + adaptive = AMR_IMPL.read_text() + for source in (uniform, adaptive): + assert "automatic_balance_due" in source + assert "impl.program_.automatic_balance_due_" in source + + +def test_projection_delta_is_captured_only_when_due_and_stays_qualified() -> None: + services = EXECUTION_SERVICES.read_text() + projection = _between( + services, + "void apply_projection(int block, MultiFab& state) const", + "/// Minimum physical cell size used by the native CFL authority.", + ) + assert "if (!runtime.automatic_balance_capture_due())" in projection + assert projection.count("program_execution_projection_balance_integrals_") == 2 + assert projection.index("const std::optional> before") < projection.index( + "program_execution_apply_projection_" + ) + assert projection.index("program_execution_apply_projection_") < projection.index( + "const std::optional> after" + ) + assert "record_automatic_balance_term(" in projection + assert '"projection"' in projection + assert "runtime_block, level, component" in projection + + state = PROGRAM_STATE.read_text() + accepted = _between( + state, + "std::map accepted_balance_terms(", + "void begin_balance_due_window(", + ) + assert "automatic_balance_terms_" not in accepted + + +def test_uniform_projection_evidence_uses_exact_available_measure() -> None: + context = UNIFORM_CONTEXT.read_text() + provider = _between( + context, + "std::optional> program_execution_projection_balance_integrals_(", + "Real program_execution_hmin_() const", + ) + assert "if (sys_->program_is_polar())" in provider + assert "return std::nullopt;" in provider + assert "context.geom.dx() * context.geom.dy()" in provider + assert "RelativeCellMeasure measure;" in provider + assert "measure.active_cells = context.domain_mask;" in provider + assert "measure.inverse_volume_fraction = context.eb_inverse_volume_fraction;" in provider + assert "pops::reduce_sum(state, component, measure)" in provider + + +def test_amr_projection_evidence_excludes_covered_cells_and_reduces_once() -> None: + context = AMR_CONTEXT.read_text() + provider = _between( + context, + "std::optional> program_execution_projection_balance_integrals_(", + "Real program_execution_hmin_() const", + ) + assert "active_mask(views, level_, next)" in provider + assert "CompositeSumKind::Sum" in provider + assert "local_sum(" in provider + assert "if (!eng_->level_is_replicated(level_))" in provider + assert provider.count("all_reduce_sum_inplace(") == 1 + assert "geometry.dx()) * static_cast(geometry.dy())" in provider + assert "state.n_grow() != live.n_grow()" in provider + assert "state.local_size() != live.local_size()" in provider diff --git a/tests/python/unit/time/test_time_ops_polish.py b/tests/python/unit/time/test_time_ops_polish.py index d3f8a996c..b996b6950 100644 --- a/tests/python/unit/time/test_time_ops_polish.py +++ b/tests/python/unit/time/test_time_ops_polish.py @@ -360,6 +360,10 @@ def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): source = emit_cpp_program(P, balance_due_contract=contract) assert source.count("ctx.record_balance_term(") == 5 assert source.count("ctx.balance_consumer_is_due(") == 1 + assert source.count("ctx.note_automatic_balance_capture_due(") == 1 + assert source.index("ctx.note_automatic_balance_capture_due(") < source.index( + "ctx.record_balance_term(" + ) assert '"%s", 3)' % route.token in source assert "? (ctx.sum_component(" in source assert "ctx.record_scalar(" not in source @@ -367,12 +371,11 @@ def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): unreachable_source = emit_cpp_program( P, - balance_due_contract=_balance_due_contract( - route, every(1 << 31, clock=P.clock) - ), + balance_due_contract=_balance_due_contract(route, every(1 << 31, clock=P.clock)), ) assert "2147483648" not in unreachable_source assert "ctx.balance_consumer_is_due(" not in unreachable_source + assert "ctx.note_automatic_balance_capture_due(" not in unreachable_source def test_balance_due_contract_unions_consumers_and_ignores_static_false(t): @@ -420,6 +423,7 @@ def test_record_balance_elides_native_collectives_without_a_consumer(t): source = emit_cpp_program(P) assert "ctx.balance_consumer_is_due(" not in source + assert "ctx.note_automatic_balance_capture_due(" not in source assert "ctx.record_balance_term(" not in source assert "(false) ? (ctx.sum_component(" in source @@ -480,6 +484,7 @@ def test_record_balance_physical_time_cadence_stays_conservatively_due(t): ) assert source.count("ctx.balance_consumer_is_due(") == 1 + assert source.count("ctx.note_automatic_balance_capture_due(") == 1 assert '"%s", 1)' % route.token in source assert source.count("ctx.record_balance_term(") == 5 From b2c7a90be7b5a9283fd5fa890bbbea10fb9ea35a Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:03:03 +0200 Subject: [PATCH 083/109] docs(balance): describe qualified projection evidence (ADC-686) --- docs/design/exact-output-consumers.md | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/docs/design/exact-output-consumers.md b/docs/design/exact-output-consumers.md index 33b265125..45e587f90 100644 --- a/docs/design/exact-output-consumers.md +++ b/docs/design/exact-output-consumers.md @@ -498,12 +498,22 @@ balance reductions are not yet skipped. This fallback can add work but cannot su evidence. A zero-step run has no accepted native occurrence: its coincident start/end moment cannot publish an accepted-step consumer, including `Balance`. -This route is explicit evidence, not automatic numerical instrumentation: a Program that cannot -produce its actual reflux or projection increment cannot declare `Balance`. In particular, the -generic automatic extraction of AMR reflux/projection contributions from the internal native -operator ledgers remains separate work. On an adaptive layout the recorded values must already be -composite and coverage-corrected; an ordinary sum of every per-level state would double-count -covered coarse cells. Neither `Balance` nor `BalanceTerms` silently claims otherwise. +This public route still consumes explicit evidence: a Program that cannot produce every actual term +cannot declare `Balance`. Native operator instrumentation is deliberately kept in a separate, +qualified attempt-local mailbox until a resolved quantity selector can prove which +`BalanceLedger` route owns each block/level/component contribution. Generated code publishes the OR +of the exact due route decisions before the first Program operator; the marker is monotone for the +attempt, disabled during replay, and reset at attempt entry. Consequently off-cadence steps do not +pay for automatic operator reductions. + +That private mailbox currently captures the signed AMR reflux correction and the before/after +projection delta. Uniform Cartesian projection uses the authenticated cell measure and embedded +boundary mask; AMR projection excludes covered coarse cells and performs one component-vector +collective per participating level. Polar projection stays absent because no exact per-cell polar +volume provider exists on this path. Automatic physical-boundary flux and source evidence are also +not yet producers. None of these private values is read by `accepted_balance_terms()`, so this +instrumentation does not silently complete an authored five-term balance or widen the public +contract. Checkpoint remains a separate restart effect. These consumers do not define a checkpoint schema or reader and do not call the scientific-output manifest a restart identity. The checkpoint provider From 3247df3c5284399ff3cbbf9701451a5ce333a9f4 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:03:33 +0200 Subject: [PATCH 084/109] test(balance): distinguish projection fast path ordering (ADC-686) --- .../test_automatic_projection_balance_fence.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/python/architecture/test_automatic_projection_balance_fence.py b/tests/python/architecture/test_automatic_projection_balance_fence.py index 9c0e6e5fd..8ce73ec53 100644 --- a/tests/python/architecture/test_automatic_projection_balance_fence.py +++ b/tests/python/architecture/test_automatic_projection_balance_fence.py @@ -63,10 +63,10 @@ def test_projection_delta_is_captured_only_when_due_and_stays_qualified() -> Non ) assert "if (!runtime.automatic_balance_capture_due())" in projection assert projection.count("program_execution_projection_balance_integrals_") == 2 - assert projection.index("const std::optional> before") < projection.index( - "program_execution_apply_projection_" - ) - assert projection.index("program_execution_apply_projection_") < projection.index( + due_projection = projection.split( + "const std::optional> before", 1 + )[1] + assert due_projection.index("program_execution_apply_projection_") < due_projection.index( "const std::optional> after" ) assert "record_automatic_balance_term(" in projection From 2824e00788d16aa8e007ef73043dca31b41c2870 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:05:00 +0200 Subject: [PATCH 085/109] release: compare the installed public API --- scripts/prove_public_api_parity.py | 113 +++++++++++++++++++++++------ 1 file changed, 92 insertions(+), 21 deletions(-) diff --git a/scripts/prove_public_api_parity.py b/scripts/prove_public_api_parity.py index 013806506..f09efe785 100644 --- a/scripts/prove_public_api_parity.py +++ b/scripts/prove_public_api_parity.py @@ -6,6 +6,7 @@ import argparse from collections.abc import Mapping, Sequence import hashlib +import importlib.metadata import json from pathlib import Path, PurePosixPath import subprocess @@ -17,7 +18,7 @@ ROOT = Path(__file__).resolve().parents[1] SOURCE_PACKAGE = ROOT / "python" / "pops" -PROOF_SCHEMA_VERSION = 1 +PROOF_SCHEMA_VERSION = 2 TYPED_PAYLOAD_SUFFIXES = (".py", ".pyi") PUBLIC_ROOT = ( "Model", @@ -174,9 +175,9 @@ def _is_typed_payload(relative: str) -> bool: return path.name == "py.typed" or path.suffix in TYPED_PAYLOAD_SUFFIXES -def _source_manifest(package: Path = SOURCE_PACKAGE) -> dict[str, str]: +def _typed_manifest(package: Path, *, label: str) -> dict[str, str]: if not package.is_dir(): - raise PublicApiParityError("source package is absent: %s" % package) + raise PublicApiParityError("%s package is absent: %s" % (label, package)) manifest = { path.relative_to(package).as_posix(): _sha256(path) for path in sorted(package.rglob("*")) @@ -186,7 +187,7 @@ def _source_manifest(package: Path = SOURCE_PACKAGE) -> dict[str, str]: } required = {"__init__.py", "_pops.pyi", "py.typed"} if not required.issubset(manifest): - raise PublicApiParityError("source package lacks its root API or typing payload") + raise PublicApiParityError("%s package lacks its root API or typing payload" % label) return manifest @@ -251,40 +252,96 @@ def _canonical_sha256(payload: Mapping[str, Any]) -> str: return _sha256_bytes(encoded) -def build_proof(wheel: Path) -> dict[str, Any]: +def _require_manifest_parity( + reference: Mapping[str, str], + candidate: Mapping[str, str], + *, + label: str, +) -> None: + if candidate == reference: + return + missing = sorted(set(reference) - set(candidate)) + extra = sorted(set(candidate) - set(reference)) + changed = sorted( + name + for name in set(reference) & set(candidate) + if reference[name] != candidate[name] + ) + raise PublicApiParityError( + "%s Python/typing payload differs from source " + "(missing=%s, extra=%s, changed=%s)" + % (label, missing[:8], extra[:8], changed[:8]) + ) + + +def _installed_package_from_distribution() -> Path: + try: + distribution = importlib.metadata.distribution("PoPS") + except importlib.metadata.PackageNotFoundError as exc: + raise PublicApiParityError("the PoPS distribution is not installed") from exc + files = distribution.files + if files is None: + raise PublicApiParityError("the installed PoPS distribution has no file inventory") + package_initializers = [ + row for row in files if PurePosixPath(str(row)).as_posix() == "pops/__init__.py" + ] + if len(package_initializers) != 1: + raise PublicApiParityError( + "the installed PoPS distribution has no unique pops/__init__.py") + package = Path(distribution.locate_file(package_initializers[0])).resolve().parent + if not package.is_dir(): + raise PublicApiParityError("the installed PoPS package directory is absent") + try: + package.relative_to(ROOT) + except ValueError: + return package + raise PublicApiParityError( + "the installed-package proof resolved inside the source checkout: %s" % package) + + +def build_proof( + wheel: Path, + *, + installed_package: Path | None = None, +) -> dict[str, Any]: """Compare one exact wheel archive with the current source checkout.""" retained = wheel.expanduser().resolve() if retained.suffix != ".whl" or not retained.is_file(): raise PublicApiParityError("release artifact is not one readable wheel") - source_manifest = _source_manifest() + source_manifest = _typed_manifest(SOURCE_PACKAGE, label="source") + installed = None if installed_package is None else installed_package.expanduser().resolve() + if installed is not None: + try: + installed.relative_to(ROOT) + except ValueError: + pass + else: + raise PublicApiParityError( + "the installed-package proof resolved inside the source checkout: %s" % installed) try: with tempfile.TemporaryDirectory(prefix="pops-public-api-") as temporary: extracted = Path(temporary) with zipfile.ZipFile(retained) as archive: wheel_manifest = _wheel_manifest(archive) - if wheel_manifest != source_manifest: - missing = sorted(set(source_manifest) - set(wheel_manifest)) - extra = sorted(set(wheel_manifest) - set(source_manifest)) - changed = sorted( - name - for name in set(source_manifest) & set(wheel_manifest) - if source_manifest[name] != wheel_manifest[name] - ) - raise PublicApiParityError( - "wheel Python/typing payload differs from source " - "(missing=%s, extra=%s, changed=%s)" - % (missing[:8], extra[:8], changed[:8]) - ) + _require_manifest_parity( + source_manifest, wheel_manifest, label="wheel") _safe_extract(archive, extracted) source_snapshot = _snapshot(SOURCE_PACKAGE.parent) wheel_snapshot = _snapshot(extracted) + if installed is not None: + installed_manifest = _typed_manifest(installed, label="installed") + _require_manifest_parity( + source_manifest, installed_manifest, label="installed") + installed_snapshot = _snapshot(installed.parent) except (OSError, zipfile.BadZipFile) as exc: raise PublicApiParityError("release wheel is unreadable: %s" % exc) from exc if wheel_snapshot != source_snapshot: raise PublicApiParityError("wheel and source public API snapshots differ") + if installed is not None and installed_snapshot != source_snapshot: + raise PublicApiParityError("installed and source public API snapshots differ") if tuple(source_snapshot["public"]) != PUBLIC_ROOT: raise PublicApiParityError("public API snapshot differs from the final root contract") - return { + proof = { "schema_version": PROOF_SCHEMA_VERSION, "wheel_path": str(retained), "wheel_sha256": _sha256(retained), @@ -295,7 +352,15 @@ def build_proof(wheel: Path) -> dict[str, Any]: "pure_authoring": source_snapshot["pure_authoring"], "qualified_handles": source_snapshot["qualified_handles"], "py_typed": source_snapshot["py_typed"], + "installed": installed is not None, } + if installed is not None: + proof.update({ + "installed_package": str(installed), + "installed_typed_payload_sha256": _canonical_sha256(installed_manifest), + "installed_public_api_sha256": _canonical_sha256(installed_snapshot), + }) + return proof def _write_evidence(path: Path, proof: Mapping[str, Any]) -> None: @@ -321,10 +386,16 @@ def _write_evidence(path: Path, proof: Mapping[str, Any]) -> None: def main(argv: Sequence[str] | None = None) -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--wheel", required=True, type=Path) + parser.add_argument( + "--installed", + action="store_true", + help="also prove the importlib.metadata-resolved installed distribution outside checkout", + ) parser.add_argument("--evidence", type=Path) args = parser.parse_args(argv) try: - proof = build_proof(args.wheel) + installed = _installed_package_from_distribution() if args.installed else None + proof = build_proof(args.wheel, installed_package=installed) if args.evidence is not None: _write_evidence(args.evidence, proof) except (PublicApiParityError, OSError, ValueError) as exc: From f6d91d2bd62b323e6fb3dd401ef3563489805a18 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:05:29 +0200 Subject: [PATCH 086/109] release: gate on installed API parity --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5ebbba136..dac1cf5f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,10 +61,11 @@ jobs: wheels=("$RUNNER_TEMP"/wheelhouse/pops-*.whl) test "${#wheels[@]}" -eq 1 evidence="$RUNNER_TEMP/pops-final-evidence.json" + python scripts/run_final_gate.py --wheel "${wheels[0]}" --evidence "$evidence" python scripts/prove_public_api_parity.py \ --wheel "${wheels[0]}" \ + --installed \ --evidence "$RUNNER_TEMP/pops-final-evidence-public-api.json" - python scripts/run_final_gate.py --wheel "${wheels[0]}" --evidence "$evidence" python - <<'PY' from pops.runtime_environment import runtime_environment_report report = runtime_environment_report() From 7e43a776c45d314370dad5338ff9a9917182437f Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:05:44 +0200 Subject: [PATCH 087/109] feat(amr): resolve Reflux through provider protocol --- python/pops/amr/__init__.py | 2 + python/pops/amr/_resolution.py | 10 ++- python/pops/amr/providers.py | 85 +++++++++++++++++++++ python/pops/codegen/_amr_plan_validation.py | 5 +- python/pops/layouts/__init__.py | 20 ++++- python/pops/lib/amr/__init__.py | 53 +++++++++++++ python/pops/runtime/_runtime_authorities.py | 6 +- 7 files changed, 171 insertions(+), 10 deletions(-) diff --git a/python/pops/amr/__init__.py b/python/pops/amr/__init__.py index 1db610322..5cea94341 100644 --- a/python/pops/amr/__init__.py +++ b/python/pops/amr/__init__.py @@ -36,6 +36,7 @@ AMRProviderLoweringContext, amr_provider_binding_identity, ClusteringProvider, + RefluxProvider, ResolvedAMRProviderBinding, TaggerProvider, ) @@ -66,6 +67,7 @@ "PatchLayout", "PreparedHierarchyNativeLowering", "PreparedHierarchyNativeProvider", + "RefluxProvider", "ResolvedAMRProviderBinding", "Tag", "TaggerProvider", diff --git a/python/pops/amr/_resolution.py b/python/pops/amr/_resolution.py index 060a63912..62d6a027b 100644 --- a/python/pops/amr/_resolution.py +++ b/python/pops/amr/_resolution.py @@ -625,6 +625,7 @@ def resolve_amr_authorities( load_balance: Any, tagger: Any, clustering: Any, + reflux: Any, context: AMRResolutionContext, ) -> ResolvedAMRAuthorities: """Resolve every adaptive-layout concern exactly once from its owning declaration.""" @@ -643,7 +644,7 @@ def resolve_amr_authorities( raise TypeError("AMR %s authority must implement %s()" % (slot, method)) if type(context) is not AMRResolutionContext: raise TypeError("AMR resolution requires an AMRResolutionContext") - providers = (tagger, clustering) + providers = (tagger, clustering, reflux) for value in providers: for method in ("inspect", "resolve_references", "lower_amr_provider"): if not callable(getattr(value, method, None)): @@ -684,10 +685,11 @@ def resolve_amr_authorities( if lowered.role in provider_bindings: raise ValueError("AMR provider roles must be unique") provider_bindings[lowered.role] = lowered.data - if set(provider_bindings) != {"clustering", "tagger"}: - raise ValueError("AMR resolution requires exact clustering and tagger provider roles") + if set(provider_bindings) != {"clustering", "tagger", "reflux"}: + raise ValueError( + "AMR resolution requires exact clustering, tagger and reflux provider roles") provider_bindings = { - role: provider_bindings[role] for role in ("clustering", "tagger") + role: provider_bindings[role] for role in ("clustering", "tagger", "reflux") } resolved_hierarchy = _hierarchy( hierarchy, diff --git a/python/pops/amr/providers.py b/python/pops/amr/providers.py index c704b9132..223cf4d14 100644 --- a/python/pops/amr/providers.py +++ b/python/pops/amr/providers.py @@ -423,6 +423,64 @@ def runtime_binding_data(self) -> dict[str, Any]: canonical_identity = runtime_binding_data +@dataclass(frozen=True, slots=True) +class RefluxProvider: + """Bind one external local Reflux table to the conservative AMR transition.""" + + component: Any + __pops_ir_immutable__ = True + + def __post_init__(self) -> None: + from pops import interfaces + + _external_component( + self.component, + interface=interfaces.Reflux, + where="RefluxProvider.component", + ) + + def resolve_references(self, resolver: Any) -> RefluxProvider: + if not callable(resolver): + raise TypeError("RefluxProvider.resolve_references requires a callable resolver") + return self + + def require_component_inputs(self, components: Any) -> None: + _require_component(self.component, components, where="RefluxProvider") + + def lower_amr_provider( + self, context: AMRProviderLoweringContext, + ) -> ResolvedAMRProviderBinding: + """Authenticate the component, hierarchy layout and Program clock.""" + if type(context) is not AMRProviderLoweringContext: + raise TypeError("RefluxProvider requires an AMRProviderLoweringContext") + self.require_component_inputs(context.components) + data = { + **self.runtime_binding_data(), + "layout_identity": context.layout_identity, + "clock_identity": context.clock_identity, + } + data["provider_identity"] = amr_provider_binding_identity("reflux", data) + return ResolvedAMRProviderBinding("reflux", data) + + def runtime_binding_data(self) -> dict[str, Any]: + from pops import interfaces + + data = { + "schema_version": 1, + "provider_type": "external_amr_reflux", + "runtime_installation": { + "schema_version": 1, + "protocol": "external_component", + }, + **_component_binding(self.component, interfaces.Reflux), + } + data["provider_identity"] = make_identity("amr-reflux-provider", data).token + return data + + inspect = runtime_binding_data + canonical_identity = runtime_binding_data + + @dataclass(frozen=True, slots=True) class _AMRRuntimeInterfaceProtocol: """Native-interface-owned validation and installation route.""" @@ -598,6 +656,26 @@ def validate_installed_capability( "external AMR Tagger lacks its exact graph/capability/clock contract") +@dataclass(frozen=True, slots=True) +class _RefluxRuntimeInterfaceProtocol(_AMRRuntimeInterfaceProtocol): + """The local Reflux callback is qualified by the accepted Program clock.""" + + def validate_resolved_capability( + self, binding: Mapping[str, Any], resolved_tagging_identity: str | None, + ) -> None: + del resolved_tagging_identity + if not isinstance(binding.get("clock_identity"), str) \ + or not binding["clock_identity"]: + raise ValueError("AMR Reflux lacks its exact Program clock authority") + + def validate_installed_capability( + self, binding: Mapping[str, Any], installed: Any, + resolved_tagging_identity: str | None, + ) -> None: + del installed + self.validate_resolved_capability(binding, resolved_tagging_identity) + + def _runtime_interface_key(value: Any) -> tuple[Any, ...]: if not isinstance(value, Mapping): raise TypeError("AMR provider binding has no native-interface protocol") @@ -632,6 +710,12 @@ def _runtime_interface_protocols() -> dict[tuple[Any, ...], _AMRRuntimeInterface builtin_provider_id="pops.lib.amr::symbolic_tagger", component_installer="_install_amr_tagger_component", ), + _RefluxRuntimeInterfaceProtocol( + role="reflux", + native_interface=interfaces.Reflux.to_data(), + builtin_provider_id="pops.lib.amr::flux_register_reflux", + component_installer="_install_amr_reflux_component", + ), ) return {_runtime_interface_key(row.native_interface): row for row in protocols} @@ -933,6 +1017,7 @@ def prepare_amr_provider_installation( "amr_provider_binding_identity", "ClusteringProvider", "PreparedAMRProviderNativeConfig", + "RefluxProvider", "ResolvedAMRProviderBinding", "TaggerProvider", "validate_amr_provider_binding", diff --git a/python/pops/codegen/_amr_plan_validation.py b/python/pops/codegen/_amr_plan_validation.py index 63d9a8311..29fe06fd2 100644 --- a/python/pops/codegen/_amr_plan_validation.py +++ b/python/pops/codegen/_amr_plan_validation.py @@ -81,8 +81,9 @@ def validate_amr_authorities(plan: Any) -> None: or plan.bootstrap_plan.initial_identity != plan.initial_condition_plan.identity: raise ValueError("ResolvedSimulationPlan bootstrap does not authenticate AMR authorities") providers = plan.amr_providers - if tuple(providers) != ("clustering", "tagger"): - raise ValueError("AMR plan requires exact clustering and tagger provider bindings") + if tuple(providers) != ("clustering", "tagger", "reflux"): + raise ValueError( + "AMR plan requires exact clustering, tagger and reflux provider bindings") # Component inputs deliberately admit both source authorities and already-compiled # artifacts. Their representations differ, but both expose the same authenticated # projection protocol. Index that projection instead of reaching through the source-only diff --git a/python/pops/layouts/__init__.py b/python/pops/layouts/__init__.py index 6f8354777..b137a558e 100644 --- a/python/pops/layouts/__init__.py +++ b/python/pops/layouts/__init__.py @@ -396,6 +396,7 @@ def __init__( load_balance: Any = None, tagger: Any = None, clustering: Any = None, + reflux: Any = None, ) -> None: # Structural snapshots consume ``options()``. Keeping authorities private prevents the # generic snapshotter from recursively treating Schedule implementation helpers as public @@ -407,15 +408,22 @@ def __init__( self._transfer = transfer self._execution = execution self._patch_layout = PatchLayout() if patch_layout is None else patch_layout - if load_balance is None or tagger is None or clustering is None: - from pops.lib.amr import BergerRigoutsos, SpaceFillingCurve, SymbolicTagger + if load_balance is None or tagger is None or clustering is None or reflux is None: + from pops.lib.amr import ( + BergerRigoutsos, + FluxRegisterReflux, + SpaceFillingCurve, + SymbolicTagger, + ) load_balance = SpaceFillingCurve() if load_balance is None else load_balance tagger = SymbolicTagger() if tagger is None else tagger clustering = BergerRigoutsos() if clustering is None else clustering + reflux = FluxRegisterReflux() if reflux is None else reflux self._load_balance = load_balance self._tagger = tagger self._clustering = clustering + self._reflux = reflux @property def grid(self) -> Any: @@ -457,6 +465,10 @@ def tagger(self) -> Any: def clustering(self) -> Any: return self._clustering + @property + def reflux(self) -> Any: + return self._reflux + def _validate_authorities(self) -> None: authorities = { "hierarchy": self.hierarchy, "tagging": self.tagging, @@ -468,6 +480,7 @@ def _validate_authorities(self) -> None: _load_balance_data(self.load_balance) _provider_data(self.tagger, "tagger") _provider_data(self.clustering, "clustering") + _provider_data(self.reflux, "reflux") for method in ("validate", "capabilities", "requirements", "options", "to_dict"): if not callable(getattr(self.grid, method, None)): raise TypeError("AMR.grid must implement %s()" % method) @@ -516,6 +529,7 @@ def options(self) -> dict[str, Any]: "load_balance": _load_balance_data(self.load_balance), "tagger": self.tagger.inspect(), "clustering": self.clustering.inspect(), + "reflux": self.reflux.inspect(), } def _summary(self) -> str: @@ -567,6 +581,7 @@ def resolved(value: Any) -> Any: load_balance=self.load_balance, tagger=self.tagger.resolve_references(resolved), clustering=self.clustering.resolve_references(resolved), + reflux=self.reflux.resolve_references(resolved), ) def resolve_amr_authorities(self, context: Any) -> Any: @@ -583,6 +598,7 @@ def resolve_amr_authorities(self, context: Any) -> Any: load_balance=self.load_balance, tagger=self.tagger, clustering=self.clustering, + reflux=self.reflux, context=context, ) diff --git a/python/pops/lib/amr/__init__.py b/python/pops/lib/amr/__init__.py index e59531afa..4fc244028 100644 --- a/python/pops/lib/amr/__init__.py +++ b/python/pops/lib/amr/__init__.py @@ -373,6 +373,58 @@ def runtime_binding_data(self) -> dict[str, Any]: canonical_identity = runtime_binding_data +@dataclass(frozen=True, slots=True) +class FluxRegisterReflux: + """Builtin conservative flux-register correction through the Reflux provider protocol.""" + + __pops_ir_immutable__: ClassVar[bool] = True + + def resolve_references(self, resolver: Any) -> FluxRegisterReflux: + if not callable(resolver): + raise TypeError("FluxRegisterReflux.resolve_references requires a callable resolver") + return self + + def require_component_inputs(self, components: Any) -> None: + del components + + def lower_amr_provider(self, context: Any) -> Any: + from pops.amr.providers import ( + AMRProviderLoweringContext, + ResolvedAMRProviderBinding, + amr_provider_binding_identity, + ) + + if type(context) is not AMRProviderLoweringContext: + raise TypeError("FluxRegisterReflux requires an AMRProviderLoweringContext") + self.require_component_inputs(context.components) + data = { + **self.runtime_binding_data(), + "layout_identity": context.layout_identity, + "clock_identity": context.clock_identity, + } + data["provider_identity"] = amr_provider_binding_identity("reflux", data) + return ResolvedAMRProviderBinding("reflux", data) + + def runtime_binding_data(self) -> dict[str, Any]: + from pops import interfaces + + data = { + "schema_version": 1, + "provider_type": "builtin_amr_reflux", + "runtime_installation": { + "schema_version": 1, + "protocol": "builtin", + }, + "provider_id": "pops.lib.amr::flux_register_reflux", + "native_interface": interfaces.Reflux.to_data(), + } + data["provider_identity"] = make_identity("amr-reflux-provider", data).token + return data + + inspect = runtime_binding_data + canonical_identity = runtime_binding_data + + @dataclass(frozen=True, slots=True) class BergerRigoutsos: """Builtin clustering provider with intrinsic validated algorithm controls.""" @@ -459,6 +511,7 @@ def runtime_binding_data(self) -> dict[str, Any]: "DivergencePreservingFace", "EllipticRecompute", "FaceTransfer", + "FluxRegisterReflux", "LinearTimeInterpolation", "Knapsack", "NodeTransfer", diff --git a/python/pops/runtime/_runtime_authorities.py b/python/pops/runtime/_runtime_authorities.py index aff8a712d..7d51102f1 100644 --- a/python/pops/runtime/_runtime_authorities.py +++ b/python/pops/runtime/_runtime_authorities.py @@ -511,8 +511,10 @@ def _install_amr_provider_authorities(engine: Any, install_plan: Any) -> None: """Install every AMR provider through its authority-carried runtime protocol.""" providers = install_plan.amr_providers - if not isinstance(providers, Mapping) or tuple(providers) != ("clustering", "tagger"): - raise ValueError("adaptive runtime requires exact clustering and tagger providers") + if not isinstance(providers, Mapping) \ + or tuple(providers) != ("clustering", "tagger", "reflux"): + raise ValueError( + "adaptive runtime requires exact clustering, tagger and reflux providers") native = getattr(engine, "_s", None) from pops.amr.providers import prepare_amr_provider_installation from pops.runtime._component_execution_context import component_execution_data From 4e7cc59ddd08e0dfd42000888aa81a93e322913f Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:05:50 +0200 Subject: [PATCH 088/109] tests(amr): prove public Reflux provider installation --- ...prepared_reflux_runtime_execution_fence.py | 7 ++- .../unit/amr/test_external_amr_providers.py | 56 +++++++++++++++---- .../unit/amr/test_public_amr_resolution.py | 13 +++++ 3 files changed, 64 insertions(+), 12 deletions(-) diff --git a/tests/python/architecture/test_prepared_reflux_runtime_execution_fence.py b/tests/python/architecture/test_prepared_reflux_runtime_execution_fence.py index 9616dbb72..88b47e4f5 100644 --- a/tests/python/architecture/test_prepared_reflux_runtime_execution_fence.py +++ b/tests/python/architecture/test_prepared_reflux_runtime_execution_fence.py @@ -26,6 +26,7 @@ AMR_SYSTEM = ROOT / "src" / "runtime" / "amr" / "amr_system.cpp" AMR_BINDING = ROOT / "python" / "bindings" / "core" / "init" / "init_amr.cpp" RUNTIME_AUTHORITIES = ROOT / "python" / "pops" / "runtime" / "_runtime_authorities.py" +AMR_PROVIDER_PROTOCOLS = ROOT / "python" / "pops" / "amr" / "providers.py" def _between(text: str, begin: str, end: str) -> str: @@ -116,10 +117,11 @@ def test_runtime_installation_reprepares_transitions_and_routes_logical_time() - ) -def test_internal_install_seam_exists_without_claiming_public_resolution() -> None: +def test_reflux_uses_the_public_normalized_amr_provider_resolution() -> None: system = AMR_SYSTEM.read_text() binding = AMR_BINDING.read_text() authorities = RUNTIME_AUTHORITIES.read_text() + protocols = AMR_PROVIDER_PROTOCOLS.read_text() assert "install_amr_reflux_component(" in system assert "runtime->install_external_reflux(amr_reflux_component_);" in system assert "if (amr_reflux_component_)" not in _between( @@ -128,5 +130,6 @@ def test_internal_install_seam_exists_without_claiming_public_resolution() -> No "if (!boundary_plans_.empty())", ) assert '"_install_amr_reflux_component"' in binding + assert 'component_installer="_install_amr_reflux_component"' in protocols assert '"_install_amr_reflux_component"' not in authorities - assert 'tuple(providers) != ("clustering", "tagger")' in authorities + assert 'tuple(providers) != ("clustering", "tagger", "reflux")' in authorities diff --git a/tests/python/unit/amr/test_external_amr_providers.py b/tests/python/unit/amr/test_external_amr_providers.py index de1364fd5..8df244749 100644 --- a/tests/python/unit/amr/test_external_amr_providers.py +++ b/tests/python/unit/amr/test_external_amr_providers.py @@ -12,7 +12,7 @@ import pytest from pops import interfaces -from pops.amr import ClusteringProvider, TaggerProvider +from pops.amr import ClusteringProvider, RefluxProvider, TaggerProvider from pops.external import build_source_package_manifest, load from pops.layouts import AMR from pops.model import ComponentManifest @@ -141,13 +141,14 @@ def test_external_tagger_native_backend_accepts_an_exact_gpu_target(tmp_path): TaggerProvider(mismatched) -def _layout(authored, *, tagger, clustering): +def _layout(authored, *, tagger, clustering, reflux=None): return AMR( grid=authored.grid, hierarchy=authored.hierarchy, tagging=authored.tagging, tagger=tagger, clustering=clustering, + reflux=authored.reflux if reflux is None else reflux, regrid=authored.regrid, transfer=authored.transfer, execution=authored.execution, @@ -160,21 +161,25 @@ def test_external_amr_providers_survive_resolution_with_exact_components(tmp_pat tmp_path, name="tagger", interface=interfaces.Tagger) clustering_component = _component( tmp_path, name="clustering", interface=interfaces.Clustering) + reflux_component = _component( + tmp_path, name="reflux", interface=interfaces.Reflux) layout = _layout( target.layout, tagger=TaggerProvider(tagger_component), clustering=ClusteringProvider(clustering_component), + reflux=RefluxProvider(reflux_component), ) resolved = pops.resolve( pops.validate(target.authoring.case), layout=layout, - components=(tagger_component, clustering_component), + components=(tagger_component, clustering_component, reflux_component), ) - assert tuple(resolved.amr_providers) == ("clustering", "tagger") + assert tuple(resolved.amr_providers) == ("clustering", "tagger", "reflux") tagger = resolved.amr_providers["tagger"] clustering = resolved.amr_providers["clustering"] + reflux = resolved.amr_providers["reflux"] assert tagger["provider_type"] == "external_amr_tagger" assert tagger["component_id"] == tagger_component.component_manifest.component_id assert tagger["tagging_graph_identity"] == resolved.bootstrap_plan.tagging.qualified_id @@ -184,6 +189,10 @@ def test_external_amr_providers_survive_resolution_with_exact_components(tmp_pat assert clustering["provider_type"] == "external_amr_clustering" assert clustering["component_id"] == clustering_component.component_manifest.component_id assert clustering["native_interface"] == interfaces.Clustering.to_data() + assert reflux["provider_type"] == "external_amr_reflux" + assert reflux["component_id"] == reflux_component.component_manifest.component_id + assert reflux["native_interface"] == interfaces.Reflux.to_data() + assert reflux["clock_identity"] == target.authoring.program.clock.qualified_id from pops.identity.semantic import semantic_value assert resolved.resolved_hierarchy.plan.clustering.options.to_data() == { @@ -465,18 +474,21 @@ def binding(slot, interface, component_id, manifest): "interface_version": interface.version, "layout_identity": layout_identity, } + if slot in {"tagger", "reflux"}: + row["clock_identity"] = clock_identity if slot == "tagger": row.update({ - "clock_identity": clock_identity, "tagging_graph_identity": graph_identity, "tagging_capability": normalized_capability, }) row["provider_identity"] = amr_provider_binding_identity(slot, row) return row - tagger_handle, clustering_handle = object(), object() - tagger_id, clustering_id = "test::tagger", "test::clustering" - tagger_manifest, clustering_manifest = "manifest::tagger", "manifest::clustering" + tagger_handle, clustering_handle, reflux_handle = object(), object(), object() + tagger_id, clustering_id, reflux_id = "test::tagger", "test::clustering", "test::reflux" + tagger_manifest = "manifest::tagger" + clustering_manifest = "manifest::clustering" + reflux_manifest = "manifest::reflux" installed = { tagger_id: SimpleNamespace( component_manifest=SimpleNamespace(token=tagger_manifest), @@ -490,6 +502,12 @@ def binding(slot, interface, component_id, manifest): native_handle=clustering_handle, runtime_contract=SimpleNamespace(capabilities=()), ), + reflux_id: SimpleNamespace( + component_manifest=SimpleNamespace(token=reflux_manifest), + interface=interfaces.Reflux, + native_handle=reflux_handle, + runtime_contract=SimpleNamespace(capabilities=()), + ), } execution = ExecutionContext( backend=proven_serial_manifest( @@ -506,6 +524,8 @@ def binding(slot, interface, component_id, manifest): clustering_id, clustering_manifest), "tagger": binding( "tagger", interfaces.Tagger, tagger_id, tagger_manifest), + "reflux": binding( + "reflux", interfaces.Reflux, reflux_id, reflux_manifest), }, components=installed, execution_context=execution, @@ -528,6 +548,9 @@ def _install_amr_clustering_component(self, *args): def _install_amr_tagger_component(self, *args): self.calls.append(("tagger", args)) + def _install_amr_reflux_component(self, *args): + self.calls.append(("reflux", args)) + def _discard_amr_provider_components(self): self.calls.clear() self.discarded = True @@ -535,10 +558,11 @@ def _discard_amr_provider_components(self): native = Native() engine = SimpleNamespace(_s=native) _install_amr_provider_authorities(engine, plan) - assert [name for name, _ in native.calls] == ["clustering", "tagger"] + assert [name for name, _ in native.calls] == ["clustering", "tagger", "reflux"] assert native.calls[0][1][0] is clustering_handle assert native.calls[1][1][0] is tagger_handle - assert tuple(engine._amr_provider_authorities) == ("clustering", "tagger") + assert native.calls[2][1][0] is reflux_handle + assert tuple(engine._amr_provider_authorities) == ("clustering", "tagger", "reflux") missing = SimpleNamespace(**vars(plan)) missing.components = {clustering_id: installed[clustering_id]} @@ -547,3 +571,15 @@ def _discard_amr_provider_components(self): _install_amr_provider_authorities(SimpleNamespace(_s=untouched), missing) assert untouched.calls == [] assert not untouched.discarded + + missing_reflux = SimpleNamespace(**vars(plan)) + missing_reflux.components = { + clustering_id: installed[clustering_id], + tagger_id: installed[tagger_id], + } + untouched_reflux = Native() + with pytest.raises(ValueError, match="AMR reflux provider.*not installed"): + _install_amr_provider_authorities( + SimpleNamespace(_s=untouched_reflux), missing_reflux) + assert untouched_reflux.calls == [] + assert not untouched_reflux.discarded diff --git a/tests/python/unit/amr/test_public_amr_resolution.py b/tests/python/unit/amr/test_public_amr_resolution.py index 8d599742b..39fabc5ab 100644 --- a/tests/python/unit/amr/test_public_amr_resolution.py +++ b/tests/python/unit/amr/test_public_amr_resolution.py @@ -590,6 +590,19 @@ def set_temporal_relations(self, numerators, denominators, policies): "memory_spaces": list(tagging_abi["memory_spaces"]), }, }, + "reflux": { + "schema_version": 1, + "provider_type": "builtin_amr_reflux", + "runtime_installation": { + "schema_version": 1, + "protocol": "builtin", + }, + "provider_id": "pops.lib.amr::flux_register_reflux", + "provider_identity": "test::reflux-provider", + "native_interface": interfaces.Reflux.to_data(), + "layout_identity": layout_identity, + "clock_identity": "test::clock", + }, }, ) for role, binding in install_plan.amr_providers.items(): From bd8ee62e30eb9d26f174c7d9baf3d5c531613ece Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:05:56 +0200 Subject: [PATCH 089/109] docs(amr): publish Reflux provider selection --- CHANGELOG.md | 3 +++ ...TION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md | 21 +++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89338adb0..3aa6fe1fc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning ### Changed +- External AMR `Reflux` components now use the normalized public provider route from + `AMR(..., reflux=...)` through resolve, compiled provenance and transactional native + installation; the builtin flux-register kernel follows the same reported contract. - AMR checkpoint capability reports now distinguish same-rank bit-identical replay from non-bit-identical rank-count rematerialization with Dense persisted histories. The M3 gate executes the persisted two-rank to one-rank restart proof. The explicit `RegridOnRestart()` diff --git a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md index 935aac84b..1ce2877ed 100644 --- a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md +++ b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md @@ -606,22 +606,24 @@ Les builtins de `pops.lib.amr` et les composants externes implémentent le même provider. Un composant externe est sélectionné sans callback Python : ```python -from pops.amr import ClusteringProvider, TaggerProvider +from pops.amr import ClusteringProvider, RefluxProvider, TaggerProvider layout = AMR( ..., tagger=TaggerProvider(component=my_tagger), clustering=ClusteringProvider(component=my_clustering), + reflux=RefluxProvider(component=my_reflux), ) resolved = pops.resolve( pops.validate(case), layout=layout, - components=(my_tagger, my_clustering), + components=(my_tagger, my_clustering, my_reflux), ) ``` -Les deux valeurs doivent référencer un exact `pops.external.ExternalComponent` portant -respectivement l'interface générée `Tagger` ou `Clustering`. Le même objet exact doit être fourni à +Les trois valeurs doivent référencer un exact `pops.external.ExternalComponent` portant +respectivement l'interface générée `Tagger`, `Clustering` ou `Reflux`. Le même objet exact doit +être fourni à `resolve(components=...)`; son identité de manifest, son interface et sa version traversent `resolve -> compile -> bind`. Le manifest doit déclarer une classification déterministe `bitwise` ou `reproducible`, car chaque rang doit produire la même hiérarchie. Un `Tagger` déclare en plus une @@ -1426,11 +1428,12 @@ d'échec entre rangs, puis applique seul périodicité, masque de couverture, r publication transactionnelle. La présence et le contrat exact du provider sont également comparés entre rangs avant toute exécution. -Cette tranche ne publie pas encore la sélection `Reflux` dans la résolution normalisée des providers -AMR : le seam d'installation demeure interne et les configurations publiques continuent donc -d'utiliser le kernel builtin. La qualification initiale de l'adaptateur reste limitée à la cible 2D, -`float64`, CPU avec stockage hôte. Le chemin n'est pas encore prouvé par compilation native, exécution -MPI avec un composant externe, mesure de conservation ni backend GPU. +La sélection `AMR(..., reflux=RefluxProvider(component))` traverse désormais la même résolution +normalisée, identité de provider, artifact et transaction d'installation que `Tagger` et +`Clustering`. Sans sélection explicite, `FluxRegisterReflux` décrit le kernel builtin par le même +protocole et apparaît dans le même rapport de providers. La qualification initiale de l'adaptateur +reste limitée à la cible 2D, `float64`, CPU avec stockage hôte. Le chemin n'est pas encore prouvé par +exécution MPI avec un composant externe, mesure de conservation ni backend GPU. Les champs sémantiques inconnus, capacités sans preuve, collisions d'identité et entry points manquants sont refusés. Un vieux manifest n'est pas « réparé » silencieusement. From 220cefef82ed61742f9a8817245a1f6d2caaf885 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Thu, 30 Jul 2026 08:06:06 +0200 Subject: [PATCH 090/109] test(release): reject installed API drift --- .../test_public_api_parity_proof.py | 44 ++++++++++++++++--- 1 file changed, 39 insertions(+), 5 deletions(-) diff --git a/tests/python/architecture/test_public_api_parity_proof.py b/tests/python/architecture/test_public_api_parity_proof.py index 53af6f9c8..9143fba69 100644 --- a/tests/python/architecture/test_public_api_parity_proof.py +++ b/tests/python/architecture/test_public_api_parity_proof.py @@ -4,6 +4,7 @@ import importlib.util from pathlib import Path +import shutil import sys import zipfile @@ -41,26 +42,57 @@ def _synthetic_wheel(path: Path, *, omit: str | None = None) -> None: ) +def _installed_package(root: Path) -> Path: + package = root / "site-packages" / "pops" + shutil.copytree( + proof.SOURCE_PACKAGE, + package, + ignore=shutil.ignore_patterns("__pycache__", "*.pyc"), + ) + return package + + def test_exact_wheel_and_source_share_public_api_typing_and_lazy_authoring(tmp_path): wheel = tmp_path / "pops-1.0.0-py3-none-any.whl" _synthetic_wheel(wheel) + installed = _installed_package(tmp_path) - evidence = proof.build_proof(wheel) + evidence = proof.build_proof(wheel, installed_package=installed) - assert evidence["schema_version"] == 1 + assert evidence["schema_version"] == 2 assert evidence["public_names"] == list(proof.PUBLIC_ROOT) assert evidence["pure_authoring"] is True assert evidence["qualified_handles"] is True assert evidence["py_typed"] is True assert evidence["typed_payload_files"] > 100 + assert evidence["installed"] is True + assert evidence["installed_package"] == str(installed.resolve()) + assert evidence["installed_typed_payload_sha256"] == evidence["typed_payload_sha256"] + assert evidence["installed_public_api_sha256"] == evidence["public_api_sha256"] def test_wheel_proof_fails_closed_when_typing_payload_is_missing(tmp_path): wheel = tmp_path / "pops-1.0.0-py3-none-any.whl" _synthetic_wheel(wheel, omit="_pops.pyi") + installed = _installed_package(tmp_path) with pytest.raises(proof.PublicApiParityError, match="typing payload"): - proof.build_proof(wheel) + proof.build_proof(wheel, installed_package=installed) + + +def test_installed_proof_rejects_payload_drift_and_source_checkout_alias(tmp_path): + wheel = tmp_path / "pops-1.0.0-py3-none-any.whl" + _synthetic_wheel(wheel) + installed = _installed_package(tmp_path) + (installed / "__init__.py").write_text( + (installed / "__init__.py").read_text(encoding="utf-8") + "\nDRIFT = True\n", + encoding="utf-8", + ) + + with pytest.raises(proof.PublicApiParityError, match="installed Python/typing payload"): + proof.build_proof(wheel, installed_package=installed) + with pytest.raises(proof.PublicApiParityError, match="inside the source checkout"): + proof.build_proof(wheel, installed_package=proof.SOURCE_PACKAGE) def test_release_workflow_blocks_publication_on_source_wheel_api_parity(): @@ -71,7 +103,9 @@ def test_release_workflow_blocks_publication_on_source_wheel_api_parity(): assert "scripts/prove_public_api_parity.py" in validate assert '--wheel "${wheels[0]}"' in validate + assert "--installed" in validate assert 'pops-final-evidence-public-api.json' in validate + assert validate.index("scripts/run_final_gate.py") < validate.index( + "scripts/prove_public_api_parity.py") assert validate.index("scripts/prove_public_api_parity.py") < validate.index( - "scripts/run_final_gate.py" - ) + "scripts/release_preflight.py") From 9795d671aeadc373c3aea168a1c0d62153a1ca4c Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:22:45 +0200 Subject: [PATCH 091/109] test(release): exercise installed API resolution --- .../test_public_api_parity_proof.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/tests/python/architecture/test_public_api_parity_proof.py b/tests/python/architecture/test_public_api_parity_proof.py index 9143fba69..cda32be7d 100644 --- a/tests/python/architecture/test_public_api_parity_proof.py +++ b/tests/python/architecture/test_public_api_parity_proof.py @@ -3,8 +3,11 @@ from __future__ import annotations import importlib.util +import json +import os from pathlib import Path import shutil +import subprocess import sys import zipfile @@ -52,6 +55,21 @@ def _installed_package(root: Path) -> Path: return package +def _installed_distribution(root: Path) -> Path: + package = _installed_package(root) + distribution = package.parent / "pops-1.0.0.dist-info" + distribution.mkdir() + (distribution / "METADATA").write_text( + "Metadata-Version: 2.3\nName: PoPS\nVersion: 1.0.0\n", + encoding="utf-8", + ) + (distribution / "RECORD").write_text( + "pops/__init__.py,,\n", + encoding="utf-8", + ) + return package + + def test_exact_wheel_and_source_share_public_api_typing_and_lazy_authoring(tmp_path): wheel = tmp_path / "pops-1.0.0-py3-none-any.whl" _synthetic_wheel(wheel) @@ -95,6 +113,43 @@ def test_installed_proof_rejects_payload_drift_and_source_checkout_alias(tmp_pat proof.build_proof(wheel, installed_package=proof.SOURCE_PACKAGE) +def test_installed_cli_resolves_distribution_after_install_without_checkout_shadowing( + tmp_path, +): + wheel = tmp_path / "pops-1.0.0-py3-none-any.whl" + _synthetic_wheel(wheel) + installed = _installed_distribution(tmp_path) + evidence = tmp_path / "installed-public-api.json" + environment = os.environ.copy() + environment["PYTHONPATH"] = str(installed.parent) + environment["PYTHONDONTWRITEBYTECODE"] = "1" + + completed = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--wheel", + str(wheel), + "--installed", + "--evidence", + str(evidence), + ], + cwd=tmp_path, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + + assert completed.returncode == 0, completed.stdout + payload = json.loads(evidence.read_text(encoding="utf-8")) + assert payload["installed"] is True + assert payload["installed_package"] == str(installed.resolve()) + assert payload["installed_typed_payload_sha256"] == payload["typed_payload_sha256"] + assert payload["installed_public_api_sha256"] == payload["public_api_sha256"] + + def test_release_workflow_blocks_publication_on_source_wheel_api_parity(): workflow = (ROOT / ".github" / "workflows" / "release.yml").read_text( encoding="utf-8" From 1c16d32a79a37f00e07ba5a7daedb628789d28a6 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:23:10 +0200 Subject: [PATCH 092/109] docs(release): describe installed API parity gate --- ...TION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md index 5633c7c83..83be821cf 100644 --- a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md +++ b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md @@ -1536,14 +1536,19 @@ dans `examples/final/`. Chaque script doit : ## 14. Gate de conformance finale -Le job de release commence par -`scripts/prove_public_api_parity.py --wheel --evidence `. -Cette preuve compare octet par octet tous les fichiers Python et de typage (`*.py`, `*.pyi`, -`py.typed`) du checkout et du wheel retenu, puis importe séparément les deux arbres dans des -interpréteurs isolés. Les deux snapshots doivent exposer la même racine publique, les mêmes -signatures et annotations, un `Case` explicite, des handles qualifiés distincts et +Le job de release exécute d'abord +`scripts/run_final_gate.py --wheel --evidence `. Ce gate installe +l'artefact exact avant que +`scripts/prove_public_api_parity.py --wheel --installed --evidence ` +ne résolve la distribution installée avec `importlib.metadata`, sans importer `pops` dans le +processus du gate. Le chemin résolu doit être extérieur au checkout. La preuve compare octet par +octet tous les fichiers Python et de typage (`*.py`, `*.pyi`, `py.typed`) du checkout, du wheel +retenu et du package installé, puis importe séparément les trois arbres dans des interpréteurs +isolés. Les trois snapshots doivent exposer la même racine publique, les mêmes signatures et +annotations, un `Case` explicite, des handles qualifiés distincts et authoring/validation/inspection sans chargement de `_pops`. Un ancien nom public, un fichier de -typage absent ou une divergence source/wheel bloque la publication. +typage absent, un chemin provenant du checkout ou une divergence source/wheel/installé bloque la +publication. Une release ne peut être déclarée conforme que par `scripts/run_final_gate.py --evidence `. La commande exige un checkout propre, From f0900f951fda7c03143902f25bd0f03cd6f402a1 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:40:25 +0200 Subject: [PATCH 093/109] release: bind retained wheel to promised lane --- scripts/release_preflight.py | 49 ++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index 57820f96c..2e25a429c 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -190,6 +190,54 @@ def _artifact_file(root: Path, relative: Any, digest: Any, *, label: str) -> Non raise PreflightError("release evidence %s hash drifted" % label) +def _wheel_lane_contract(path: Path, archive: zipfile.ZipFile, contract: Any) -> None: + """Require one native wheel whose filename and WHEEL tags match the promised lane.""" + + lanes = contract.SUPPORTED_MATRIX["wheels"] + if len(lanes) != 1: + raise PreflightError("release contract must promise exactly one wheel lane") + lane = lanes[0] + if set(lane) != {"os", "arch", "python", "backend"}: + raise PreflightError("promised wheel lane is malformed") + if lane["os"] != "macos" or lane["arch"] != "arm64" \ + or lane["backend"] != "Kokkos Serial": + raise PreflightError("promised wheel lane has no release tag verifier") + + if path.suffix != ".whl": + raise PreflightError("release artifact is not a wheel") + parts = path.name[:-4].split("-") + if len(parts) != 5: + raise PreflightError("release wheel filename must not contain a build tag") + distribution, version, python_tag, abi_tag, platform_tag = parts + expected_python = lane["python"] + if distribution.lower().replace("_", "-") != "pops" \ + or version != contract.PACKAGE_VERSION: + raise PreflightError("release wheel filename name/version disagrees with the contract") + if python_tag != expected_python or abi_tag != expected_python: + raise PreflightError("release wheel Python/ABI tags disagree with the promised lane") + if re.fullmatch(r"macosx_\d+_\d+_arm64", platform_tag) is None: + raise PreflightError("release wheel platform tag disagrees with the promised lane") + + dist_info = "%s-%s.dist-info" % (distribution, version) + wheel_names = [name for name in archive.namelist() if name.endswith(".dist-info/WHEEL")] + if wheel_names != [dist_info + "/WHEEL"]: + raise PreflightError("release wheel has no unique lane-bound WHEEL record") + try: + wheel_metadata = archive.read(wheel_names[0]).decode("utf-8") + except UnicodeDecodeError as exc: + raise PreflightError("release wheel WHEEL record is not UTF-8") from exc + fields: dict[str, list[str]] = {} + for line in wheel_metadata.splitlines(): + if ": " in line: + key, value = line.split(": ", 1) + fields.setdefault(key, []).append(value) + expected_tag = "%s-%s-%s" % (python_tag, abi_tag, platform_tag) + if fields.get("Wheel-Version") != ["1.0"] \ + or fields.get("Root-Is-Purelib") != ["false"] \ + or fields.get("Tag") != [expected_tag]: + raise PreflightError("release WHEEL metadata disagrees with the promised native lane") + + def _checkpoint_tree(path: Path) -> str: if path.is_file(): return hashlib.sha256(path.read_bytes()).hexdigest() @@ -219,6 +267,7 @@ def _wheel_evidence(directory: Path, gates: dict[str, Any], contract: Any) -> No raise PreflightError("official build wheel size drifted") try: with zipfile.ZipFile(path) as archive: + _wheel_lane_contract(path, archive, contract) metadata_names = [name for name in archive.namelist() if name.endswith(".dist-info/METADATA")] if len(metadata_names) != 1: From 02585bf9b37f53b46d8b6b8c0001d9a1af0c0407 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:40:52 +0200 Subject: [PATCH 094/109] test(release): refuse wheel lane identity drift --- .../architecture/test_final_release_gate.py | 107 ++++++++++++++++-- 1 file changed, 100 insertions(+), 7 deletions(-) diff --git a/tests/python/architecture/test_final_release_gate.py b/tests/python/architecture/test_final_release_gate.py index 4f5909843..1ed44ea76 100644 --- a/tests/python/architecture/test_final_release_gate.py +++ b/tests/python/architecture/test_final_release_gate.py @@ -32,6 +32,48 @@ def _load(name: str, path: Path): example_runner = _load("_installed_example_test", SCRIPTS / "run_installed_example.py") +def _release_contract(version: str = "0.3.0"): + return type( + "ReleaseContract", + (), + { + "PACKAGE_VERSION": version, + "SUPPORTED_MATRIX": { + "wheels": ( + { + "os": "macos", + "arch": "arm64", + "python": "cp312", + "backend": "Kokkos Serial", + }, + ), + }, + }, + ) + + +def _write_release_wheel( + path: Path, + *, + version: str = "0.3.0", + tag: str = "cp312-cp312-macosx_11_0_arm64", + purelib: str = "false", +) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(path, "w") as archive: + archive.writestr( + f"pops-{version}.dist-info/METADATA", + f"Metadata-Version: 2.3\nName: PoPS\nVersion: {version}\n", + ) + archive.writestr( + f"pops-{version}.dist-info/WHEEL", + "Wheel-Version: 1.0\n" + "Generator: ADC-688 test\n" + f"Root-Is-Purelib: {purelib}\n" + f"Tag: {tag}\n", + ) + + def _write_final_source_tree(root: Path) -> None: specification = root / contract.FINAL_SPECIFICATION specification.parent.mkdir(parents=True) @@ -171,12 +213,7 @@ def test_artifact_reopen_requires_and_records_npz(tmp_path): def test_release_evidence_authenticates_the_exact_retained_wheel(tmp_path): wheel = tmp_path / "wheels" / "pops-0.3.0-cp312-cp312-macosx_11_0_arm64.whl" - wheel.parent.mkdir() - with zipfile.ZipFile(wheel, "w") as archive: - archive.writestr( - "pops-0.3.0.dist-info/METADATA", - "Metadata-Version: 2.3\nName: PoPS\nVersion: 0.3.0\n", - ) + _write_release_wheel(wheel) gates = { "official_build": { "evidence": { @@ -188,7 +225,7 @@ def test_release_evidence_authenticates_the_exact_retained_wheel(tmp_path): }, }, } - release = type("ReleaseContract", (), {"PACKAGE_VERSION": "0.3.0"}) + release = _release_contract() preflight._wheel_evidence(tmp_path, gates, release) gates["official_build"]["evidence"]["wheel"]["size"] += 1 @@ -196,6 +233,62 @@ def test_release_evidence_authenticates_the_exact_retained_wheel(tmp_path): preflight._wheel_evidence(tmp_path, gates, release) +@pytest.mark.parametrize( + ("filename", "tag", "purelib", "message"), + ( + ( + "pops-0.3.0-cp311-cp311-macosx_11_0_arm64.whl", + "cp311-cp311-macosx_11_0_arm64", + "false", + "Python/ABI tags", + ), + ( + "pops-0.3.0-cp312-cp312-macosx_11_0_universal2.whl", + "cp312-cp312-macosx_11_0_universal2", + "false", + "platform tag", + ), + ( + "pops-0.3.0-cp312-cp312-macosx_11_0_arm64.whl", + "cp311-cp311-macosx_11_0_arm64", + "false", + "WHEEL metadata", + ), + ( + "pops-0.3.0-cp312-cp312-macosx_11_0_arm64.whl", + "cp312-cp312-macosx_11_0_arm64", + "true", + "WHEEL metadata", + ), + ( + "pops-0.3.0-1-cp312-cp312-macosx_11_0_arm64.whl", + "cp312-cp312-macosx_11_0_arm64", + "false", + "build tag", + ), + ), +) +def test_release_evidence_refuses_wheel_lane_drift( + tmp_path, filename, tag, purelib, message, +): + wheel = tmp_path / "wheels" / filename + _write_release_wheel(wheel, tag=tag, purelib=purelib) + gates = { + "official_build": { + "evidence": { + "wheel": { + "path": str(wheel.relative_to(tmp_path)), + "sha256": gate._sha256(wheel), + "size": wheel.stat().st_size, + }, + }, + }, + } + + with pytest.raises(preflight.PreflightError, match=message): + preflight._wheel_evidence(tmp_path, gates, _release_contract()) + + def test_installed_wheel_proof_requires_exact_native_member_and_direct_url(tmp_path): wheel = tmp_path / "pops-0.3.0-cp312-cp312-macosx_11_0_arm64.whl" native_bytes = b"exact wheel extension" From 61a79359cbe371a84596c50445921e18125dd9de Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:41:22 +0200 Subject: [PATCH 095/109] docs(release): document exact wheel lane proof --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89338adb0..5fb281809 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -76,6 +76,8 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning native ABI, and checkpoint envelopes independently in one generated Python/C++ release contract; declares the exact source/wheel matrix; and adds a fail-closed release preflight requiring exact tag, installed native identity, clean tree, generated products, and authenticated final-gate evidence. + The retained wheel filename and its internal `WHEEL` record must also identify exactly the promised + native macOS/arm64/cp312 lane; Python/ABI, platform, purelib, build-tag, or metadata drift is refused. - ADC-633 Compiled condensed-implicit time Program (std.condensed_schur, theta=1) on the AMR hierarchy: the condensed operators run per level through AmrProgramContext::grid_context / assembly_target / assembly_source (matrix-free coefficiented apply, reconstruct, energy), so a flat hierarchy is bit-identical to the uniform Program (the emitted matrix-free BiCGStab runs on level 0 through ctx.solve_linear_matfree) and a refined hierarchy solves the tensor elliptic by the composite FAC (CompositeFacPoisson in amr_condensed_elliptic.hpp), matching the native source-stage route; the vestigial AMR deferral stubs are removed and the Spec 6 sec.20 clean_schur_program.amr.mono cell flips to green. - ADC-640 One spatial-reconstruction dispatch generator (include/pops/runtime/builders/scheme_dispatch.hpp): dispatch_limiter binds a typed LimiterRouteId to its compile-time reconstruction policy behind an X-macro plus a count-lock static_assert, so the 17 hand-written limiter ladders across the System, polar, AMR multi-block, AMR compiled and external-brick builders collapse to one dispatch_limiter call each. A forgotten limiter is now a build error (the -Werror-free tree could only warn on a missing switch arm). Same template instantiations, bit-identical. - ADC-637 condensed_schur gained a generic lowering route: the electrostatic-Lorentz linearization J = [[0, B_z], [-B_z, 0]] is authored in the DSL (pops.lib.physics.author_electrostatic_lorentz, an m.local_linear_map on the momentum subset) and the macro (route="generic") lowers the condensed tensor coefficient A = I + c*rho*(I - theta*dt*J)^-1, the fused RHS and the velocity reconstruction through the closed-form block_inverse codegen, with no coupling/schur vocabulary. Bit-identical to the retiring hand-written Schur brick over a multi-step trajectory at theta == 1 and theta == 0.5 (golden, np.array_equal): the coefficient tensor reuses block_inverse<2> (== LorentzEliminator's binv entries) and the flux/reconstruct vector applies reuse a new factored block_apply_inverse intrinsic reproducing apply_Binv's operation order. The brick route stays the default until it is retired. From c76e7c6f0db0e385cc2288d957acb6d964a267b0 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:49:46 +0200 Subject: [PATCH 096/109] feat(balance): select native ledger terms explicitly (ADC-686) --- python/pops/_balance_contract.py | 56 ++++++++++++++++++++-- python/pops/_balance_due_contract.py | 29 +++++++++-- python/pops/codegen/program_balance_due.py | 30 ++++++------ python/pops/diagnostics/measures.py | 12 ++++- python/pops/output/_consumer_contracts.py | 26 +++++++++- python/pops/time/_program/contract.py | 4 +- python/pops/time/_program/diagnostics.py | 56 ++++++++++++++++++---- 7 files changed, 176 insertions(+), 37 deletions(-) diff --git a/python/pops/_balance_contract.py b/python/pops/_balance_contract.py index 2fcf86671..14499db5a 100644 --- a/python/pops/_balance_contract.py +++ b/python/pops/_balance_contract.py @@ -31,31 +31,79 @@ def _canonical_name(value: Any, *, where: str) -> str: class BalanceLedger: """Identity joining one Program-authored discrete balance to one consumer. - The ledger does not contain values. :meth:`Program.record_balance` writes the five reduced - scalars into the current native step-attempt mailbox, while + The ledger does not contain values. :meth:`Program.record_balance` writes the explicitly + authored reduced scalars into the current native step-attempt mailbox. A ledger may delegate + ``reflux`` and/or ``projection`` to exact native operators for one typed component role, while :class:`pops.diagnostics.Balance` selects the same identity after that attempt has advanced successfully. """ name: str + role: Any = None + component: int | None = None + automatic_terms: tuple[str, ...] = () identity: Identity = field(init=False) __pops_ir_immutable__ = True def __post_init__(self) -> None: name = _canonical_name(self.name, where="BalanceLedger.name") + role = None + if self.role is not None: + from pops.physics.roles import native_role_token + + try: + role = native_role_token(self.role) + except TypeError as exc: + raise TypeError( + "BalanceLedger.role must be a typed pops.physics.roles.ComponentRole" + ) from exc + if not isinstance(self.automatic_terms, tuple): + raise TypeError("BalanceLedger.automatic_terms must be a tuple") + automatic_terms = tuple(sorted(set(self.automatic_terms))) + if len(automatic_terms) != len(self.automatic_terms): + raise ValueError("BalanceLedger.automatic_terms must be unique") + unsupported = set(automatic_terms).difference({"reflux", "projection"}) + if unsupported: + raise ValueError( + "BalanceLedger automatic native producers currently support only " + "reflux and projection; got %s" % sorted(unsupported) + ) + component = self.component + if automatic_terms and component is None: + component = 0 + if component is not None and (type(component) is not int or component < 0): + raise TypeError("BalanceLedger.component must be a non-negative int or None") object.__setattr__(self, "name", name) + object.__setattr__(self, "component", component) + object.__setattr__(self, "automatic_terms", automatic_terms) + payload: dict[str, Any] = {"schema_version": 1, "name": name} + if role is not None: + payload["role"] = role + if component is not None: + payload["component"] = component + if automatic_terms: + payload["automatic_terms"] = list(automatic_terms) object.__setattr__( self, "identity", - make_identity("balance-ledger", {"schema_version": 1, "name": name}), + make_identity("balance-ledger", payload), ) def to_data(self) -> dict[str, Any]: - return { + data = { "schema_version": 1, "name": self.name, "identity": self.identity.to_data(), } + if self.role is not None: + from pops.physics.roles import native_role_token + + data["role"] = native_role_token(self.role) + if self.component is not None: + data["component"] = self.component + if self.automatic_terms: + data["automatic_terms"] = list(self.automatic_terms) + return data def route_identity(self, block: Any) -> Identity: from pops.problem.handles import BlockHandle diff --git a/python/pops/_balance_due_contract.py b/python/pops/_balance_due_contract.py index 3052caaf7..f2bf15186 100644 --- a/python/pops/_balance_due_contract.py +++ b/python/pops/_balance_due_contract.py @@ -51,6 +51,7 @@ class BalanceDueRoute: route: Identity consumers: tuple[BalanceDueConsumer, ...] + automatic_terms: tuple[str, ...] = () def __post_init__(self) -> None: object.__setattr__( @@ -75,12 +76,21 @@ def __post_init__(self) -> None: if len(identities) != len(set(identities)): raise ValueError("BalanceDueRoute contains a duplicate consumer") object.__setattr__(self, "consumers", consumers) + if not isinstance(self.automatic_terms, tuple): + raise TypeError("BalanceDueRoute.automatic_terms must be a tuple") + if self.automatic_terms != tuple(sorted(set(self.automatic_terms))): + raise ValueError("BalanceDueRoute.automatic_terms must be sorted and unique") + if set(self.automatic_terms).difference({"reflux", "projection"}): + raise ValueError("BalanceDueRoute names an unavailable automatic balance producer") def to_data(self) -> dict[str, Any]: - return { + data = { "route": self.route.to_data(), "consumers": [value.to_data() for value in self.consumers], } + if self.automatic_terms: + data["automatic_terms"] = list(self.automatic_terms) + return data def accepted_step_periods(self) -> tuple[int, ...]: """Return exact native periods, conservatively using period one when unprovable. @@ -161,7 +171,9 @@ def from_consumer_graph(cls, graph: Any) -> BalanceDueContract: raise TypeError( "BalanceDueContract requires an exact resolved ConsumerGraph or None" ) - by_route: dict[str, tuple[Identity, list[BalanceDueConsumer]]] = {} + by_route: dict[ + str, tuple[Identity, list[BalanceDueConsumer], tuple[str, ...]] + ] = {} for manifest in graph.nodes: for quantity in manifest.diagnostic_quantities: for operation in quantity.execution["operations"]: @@ -173,15 +185,22 @@ def from_consumer_graph(cls, graph: Any) -> BalanceDueContract: "balance-ledger-route", where="accepted balance operation route", ) - existing = by_route.setdefault(route.token, (route, [])) + automatic_terms = tuple(operation.get("automatic_terms", ())) + existing = by_route.setdefault( + route.token, (route, [], automatic_terms) + ) + if existing[2] != automatic_terms: + raise ValueError( + "one balance route cannot select different automatic producers" + ) existing[1].append( BalanceDueConsumer(manifest.identity, manifest.schedule) ) return cls( graph.identity, tuple( - BalanceDueRoute(route, tuple(consumers)) - for route, consumers in by_route.values() + BalanceDueRoute(route, tuple(consumers), automatic_terms) + for route, consumers, automatic_terms in by_route.values() ), ) diff --git a/python/pops/codegen/program_balance_due.py b/python/pops/codegen/program_balance_due.py index 1a57dafe5..2d2c2336b 100644 --- a/python/pops/codegen/program_balance_due.py +++ b/python/pops/codegen/program_balance_due.py @@ -99,15 +99,6 @@ def _program_balance_records( ) by_term[term] = value record_routes[value.id] = route.token - expected = set(BALANCE_TERM_NAMES) - for route, by_term in terms.items(): - if set(by_term) != expected: - missing = sorted(expected.difference(by_term)) - extra = sorted(set(by_term).difference(expected)) - raise ValueError( - "Program balance route %s must record exactly five terms; missing=%s extra=%s" - % (route, missing, extra) - ) return operations, record_routes, terms @@ -118,13 +109,23 @@ def validate_balance_due_contract(program: Any, contract: Any) -> None: "balance due validation requires an exact BalanceDueContract" ) _operations, _records, terms = _program_balance_records(program) - missing = sorted( - row.route.token for row in contract.routes if row.route.token not in terms - ) - if missing: + failures = [] + for row in contract.routes: + expected = set(BALANCE_TERM_NAMES).difference(row.automatic_terms) + actual = set(terms.get(row.route.token, {})) + if actual != expected: + failures.append( + "%s missing=%s extra=%s" + % ( + row.route.token, + sorted(expected.difference(actual)), + sorted(actual.difference(expected)), + ) + ) + if failures: raise ValueError( "ConsumerGraph Balance routes have no Program.record_balance producer: %s" - % ", ".join(missing) + % "; ".join(failures) ) @@ -137,6 +138,7 @@ def prepare_balance_due_lowering( raise TypeError( "balance due lowering requires an exact BalanceDueContract" ) + validate_balance_due_contract(program, contract) operations, record_routes, terms = _program_balance_records(program) route_periods = { route: ( diff --git a/python/pops/diagnostics/measures.py b/python/pops/diagnostics/measures.py index bf4c36cf8..d595d5398 100644 --- a/python/pops/diagnostics/measures.py +++ b/python/pops/diagnostics/measures.py @@ -334,7 +334,7 @@ def __init__( ) if block is None: raise TypeError("Balance(block=...) requires an exact physics BlockHandle") - super().__init__(block=block, role=None, cadence=cadence) + super().__init__(block=block, role=ledger.role, cadence=cadence) self.ledger = ledger def options(self) -> dict: @@ -346,11 +346,19 @@ def diagnostic_execution(self) -> dict[str, Any]: route = self.ledger.route_identity(self.block) return { "schema_version": 1, - "role": None, + "role": _role_name(self.ledger.role), "operations": [ { **_operation("balance", "accepted_balance"), "balance_route": route.token, + **( + { + "automatic_terms": list(self.ledger.automatic_terms), + "balance_component": self.ledger.component, + } + if self.ledger.automatic_terms + else {} + ), }, ], "conservation": None, diff --git a/python/pops/output/_consumer_contracts.py b/python/pops/output/_consumer_contracts.py index 6d4f7517c..a0323c9b2 100644 --- a/python/pops/output/_consumer_contracts.py +++ b/python/pops/output/_consumer_contracts.py @@ -343,6 +343,9 @@ def _diagnostic_execution(value: Any) -> Mapping[str, Any]: expected = {"name", "reduction", "transform", "metric_weighted"} if reduction == "accepted_balance": expected.add("balance_route") + if "automatic_terms" in operation: + expected.add("automatic_terms") + expected.add("balance_component") if set(operation) != expected: raise TypeError("%s has an unknown schema" % where) name = _text(operation["name"], "%s.name" % where) @@ -373,6 +376,27 @@ def _diagnostic_execution(value: Any) -> Mapping[str, Any]: "accepted balance route must use the version-1 balance-ledger-route identity" ) row["balance_route"] = route.token + automatic_terms = operation.get("automatic_terms", ()) + if not isinstance(automatic_terms, (tuple, list)): + raise TypeError("%s.automatic_terms must be a sequence" % where) + automatic_terms = tuple(automatic_terms) + if automatic_terms != tuple(sorted(set(automatic_terms))): + raise ValueError( + "%s.automatic_terms must be sorted and unique" % where + ) + unsupported = set(automatic_terms).difference({"reflux", "projection"}) + if unsupported: + raise ValueError( + "%s.automatic_terms names an unavailable native producer" % where + ) + if automatic_terms: + row["automatic_terms"] = list(automatic_terms) + component = operation["balance_component"] + if type(component) is not int or component < 0: + raise TypeError( + "%s.balance_component must be a non-negative int" % where + ) + row["balance_component"] = component normalized.append(row) if len({row["name"] for row in normalized}) != len(normalized): raise ValueError("DiagnosticQuantity execution operation names must be unique") @@ -383,8 +407,6 @@ def _diagnostic_execution(value: Any) -> Mapping[str, Any]: raise ValueError( "accepted balance evidence must be the sole diagnostic execution operation" ) - if has_accepted_balance and role is not None: - raise ValueError("accepted balance evidence cannot select one component role") conservation = value["conservation"] normalized_conservation = None if conservation is not None: diff --git a/python/pops/time/_program/contract.py b/python/pops/time/_program/contract.py index 4e39deeaf..cc262ddf4 100644 --- a/python/pops/time/_program/contract.py +++ b/python/pops/time/_program/contract.py @@ -208,8 +208,8 @@ def record_balance( storage_change: Any, outward_boundary_flux: Any, sources: Any, - reflux: Any, - projection: Any, + reflux: Any = None, + projection: Any = None, ) -> tuple[Any, ...]: ... # --- solve / commit / board sugar (_ProgramSolve) --- diff --git a/python/pops/time/_program/diagnostics.py b/python/pops/time/_program/diagnostics.py index 75d3ab00f..16bf5a501 100644 --- a/python/pops/time/_program/diagnostics.py +++ b/python/pops/time/_program/diagnostics.py @@ -34,17 +34,19 @@ def record_balance( storage_change: Any, outward_boundary_flux: Any, sources: Any, - reflux: Any, - projection: Any, + reflux: Any = None, + projection: Any = None, ) -> tuple[ProgramValue, ...]: """Publish one exact five-term balance into the current native attempt. - Every term is a signed, time-integrated increment for this Program invocation and - must be an additive global Program reduction (sum/dot), or scalar arithmetic composed - exclusively from such reductions and exact literals. The native mailbox accumulates - these increments across cadence substeps in the same public macro-step. Raw Python values, - extrema/norm reductions, and rank-local runtime scalars are rejected. The five records are - attempt-local: a rejected step or consumer rollback cannot leave evidence for a later sample. + Every explicitly authored term is a signed, time-integrated increment for this Program + invocation and must be an additive global Program reduction (sum/dot), or scalar arithmetic + composed exclusively from such reductions and exact literals. A ledger that explicitly + delegates ``reflux`` or ``projection`` to its native producer requires the corresponding + argument to remain ``None``. The native mailbox accumulates all increments across cadence + substeps in the same public macro-step. Raw Python values, extrema/norm reductions, and + rank-local runtime scalars are rejected. The records are attempt-local: a rejected step or + consumer rollback cannot leave evidence for a later sample. """ from pops._balance_contract import ( BALANCE_TERM_NAMES, @@ -90,10 +92,47 @@ def require_reduced(value: Any, term: str, seen: set[int]) -> ProgramValue: "only from global reductions; got scalar op %r" % (term, value.op) ) + automatic = set(ledger.automatic_terms) + for name in automatic: + if supplied[name] is not None: + raise ValueError( + "record_balance %s is owned by the ledger's native automatic producer; " + "leave it as None" % name + ) terms = { name: require_reduced(supplied[name], name, set()) for name in BALANCE_TERM_NAMES + if name not in automatic } + if automatic: + expected_component = ledger.component + + def reduced_components( + value: ProgramValue, term: str, seen: set[int] + ) -> set[int]: + if value.id in seen: + return set() + seen.add(value.id) + if value.op == "reduce": + component = value.attrs.get("comp") + if value.attrs.get("kind") != "sum" or type(component) is not int: + raise ValueError( + "record_balance %s must use component-qualified sum reductions " + "when native terms are selected" % term + ) + return {component} + components: set[int] = set() + for item in value.inputs: + components.update(reduced_components(item, term, seen)) + return components + + for name, value in terms.items(): + components = reduced_components(value, name, set()) + if components != {expected_component}: + raise ValueError( + "record_balance %s selects components %s but the native ledger owns " + "component %d" % (name, sorted(components), expected_component) + ) blocks = {value.block for value in terms.values()} if None in blocks or len(blocks) != 1: raise ValueError( @@ -114,6 +153,7 @@ def require_reduced(value: Any, term: str, seen: set[int]) -> ProgramValue: terms[name].block, ) for name in BALANCE_TERM_NAMES + if name in terms ) @atomic_authoring From 5412a954ce7a3a7c16d5b67bc44b729ac67f3616 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:50:03 +0200 Subject: [PATCH 097/109] feat(runtime): resolve qualified automatic balance evidence (ADC-686) --- include/pops/runtime/amr_system.hpp | 4 + .../runtime/program/program_runtime_state.hpp | 88 +++++++++++++++++++ include/pops/runtime/system.hpp | 4 + python/bindings/core/init/init_amr.cpp | 3 + python/bindings/core/init/init_system.cpp | 3 + python/pops/_pops.pyi | 16 ++++ python/pops/runtime/_runtime_consumers.py | 74 ++++++++++++++-- src/runtime/amr/amr_system.cpp | 22 +++++ src/runtime/system/system_program.cpp | 17 ++++ 9 files changed, 224 insertions(+), 7 deletions(-) diff --git a/include/pops/runtime/amr_system.hpp b/include/pops/runtime/amr_system.hpp index 29bdfe8fe..06d7c10ab 100644 --- a/include/pops/runtime/amr_system.hpp +++ b/include/pops/runtime/amr_system.hpp @@ -903,6 +903,10 @@ class AmrSystem { /// Five current-attempt scalars for one typed balance route. RuntimeInstance calls this only /// inside its active outer accepted-step transaction; missing/stale/non-finite evidence fails. POPS_EXPORT std::map accepted_balance_terms(const std::string& route) const; + /// The same accepted route with selected attempt-local native reflux/projection producers. + POPS_EXPORT std::map selected_accepted_balance_terms( + const std::string& route, const std::string& block, int component, + const std::vector& levels, const std::vector& automatic_terms) const; POPS_EXPORT void begin_step_projection_report(); POPS_EXPORT void note_step_projection(const std::string& name); POPS_EXPORT std::vector consume_step_projections(); diff --git a/include/pops/runtime/program/program_runtime_state.hpp b/include/pops/runtime/program/program_runtime_state.hpp index 406faaeed..436a5fe3c 100644 --- a/include/pops/runtime/program/program_runtime_state.hpp +++ b/include/pops/runtime/program/program_runtime_state.hpp @@ -969,6 +969,94 @@ struct ProgramRuntimeState { return result; } + /// Resolve one public Balance route against exact native operator coordinates. + /// + /// Explicit Program records remain authoritative for every term not listed in @p automatic_terms. + /// Reflux and projection may instead be selected from the attempt-local native mailbox. The + /// selector is complete and owner-qualified: one runtime block, one conservative component and + /// the full active contiguous hierarchy. A selected producer must have published every expected + /// coordinate; missing evidence and duplicate Program/native authority fail instead of becoming + /// zero or reusing a stale value. + std::map selected_accepted_balance_terms( + const std::string& route, int runtime_block, int component, const std::vector& levels, + const std::vector& automatic_terms, const std::string& runtime) const { + static constexpr std::array kTerms{"storage_change", "outward_boundary_flux", + "sources", "reflux", "projection"}; + require_balance_route(route, runtime + "::_selected_accepted_balance_terms"); + if (runtime_block < 0 || component < 0) + throw std::invalid_argument( + runtime + "::_selected_accepted_balance_terms requires non-negative coordinates"); + if (levels.empty() || levels.front() < 0 || + std::adjacent_find(levels.begin(), levels.end(), + [](int left, int right) { return right != left + 1; }) != levels.end()) + throw std::invalid_argument( + runtime + "::_selected_accepted_balance_terms requires a non-empty contiguous hierarchy"); + if (!std::is_sorted(automatic_terms.begin(), automatic_terms.end()) || + std::adjacent_find(automatic_terms.begin(), automatic_terms.end()) != automatic_terms.end()) + throw std::invalid_argument( + runtime + "::_selected_accepted_balance_terms requires sorted unique automatic terms"); + for (const std::string& term : automatic_terms) + if (term != "reflux" && term != "projection") + throw std::invalid_argument( + runtime + "::_selected_accepted_balance_terms has no native producer for '" + term + + "'"); + + std::map result; + if (step_balance_terms_.empty() && balance_step_completed_ && !balance_program_was_due_) { + for (const char* term : kTerms) + result.emplace(term, Real(0)); + return result; + } + for (const char* term_value : kTerms) { + const std::string term = term_value; + const bool automatic = + std::binary_search(automatic_terms.begin(), automatic_terms.end(), term); + const std::string record = "pops.balance-term.v1:" + route + ":" + term; + const auto authored = step_balance_terms_.find(record); + if (!automatic) { + if (authored == step_balance_terms_.end()) + throw std::runtime_error( + runtime + + "::_selected_accepted_balance_terms: current native attempt omitted term '" + term + + "'; Program.record_balance must publish every non-automatic term"); + if (!std::isfinite(static_cast(authored->second))) + throw std::runtime_error( + runtime + + "::_selected_accepted_balance_terms: current native attempt produced " + "non-finite term '" + + term + "'"); + result.emplace(term, authored->second); + continue; + } + if (authored != step_balance_terms_.end()) + throw std::runtime_error(runtime + "::_selected_accepted_balance_terms: term '" + term + + "' has both Program and native producer authority"); + + Real value = Real(0); + const std::size_t expected = term == "reflux" ? levels.size() - 1 : levels.size(); + for (std::size_t index = 0; index < expected; ++index) { + const AutomaticBalanceKey key{runtime_block, levels[index], component, term}; + const auto found = automatic_balance_terms_.find(key); + if (found == automatic_balance_terms_.end()) + throw std::runtime_error( + runtime + "::_selected_accepted_balance_terms: native producer omitted term '" + + term + "' at level " + std::to_string(levels[index])); + if (!std::isfinite(static_cast(found->second))) + throw std::runtime_error( + runtime + + "::_selected_accepted_balance_terms: native producer returned non-finite " + "term '" + + term + "'"); + value += found->second; + } + if (!std::isfinite(static_cast(value))) + throw std::runtime_error( + runtime + "::_selected_accepted_balance_terms: native term accumulation overflowed"); + result.emplace(term, value); + } + return result; + } + void begin_balance_due_window(int accepted_macro_step, const std::string& runtime) { if (balance_due_window_active_) throw std::logic_error(runtime + " balance due window is already active"); diff --git a/include/pops/runtime/system.hpp b/include/pops/runtime/system.hpp index 579275ff9..63837952b 100644 --- a/include/pops/runtime/system.hpp +++ b/include/pops/runtime/system.hpp @@ -1240,6 +1240,10 @@ class System { /// Five current-attempt scalars for one typed balance route. RuntimeInstance calls this only /// inside its active outer accepted-step transaction; missing/stale/non-finite evidence fails. POPS_EXPORT std::map accepted_balance_terms(const std::string& route) const; + /// The same accepted route with selected attempt-local native reflux/projection producers. + POPS_EXPORT std::map selected_accepted_balance_terms( + const std::string& route, const std::string& block, int component, + const std::vector& levels, const std::vector& automatic_terms) const; POPS_EXPORT void begin_step_projection_report(); POPS_EXPORT void note_step_projection(const std::string& name); POPS_EXPORT std::vector consume_step_projections(); diff --git a/python/bindings/core/init/init_amr.cpp b/python/bindings/core/init/init_amr.cpp index cbde10c19..3336d071b 100644 --- a/python/bindings/core/init/init_amr.cpp +++ b/python/bindings/core/init/init_amr.cpp @@ -824,6 +824,9 @@ void bind_amr_program(py::class_& cls) { .def("program_diagnostic", &AmrSystem::program_diagnostic, py::arg("name")) .def("program_diagnostics", &AmrSystem::program_diagnostics) .def("_accepted_balance_terms", &AmrSystem::accepted_balance_terms, py::arg("route")) + .def("_selected_accepted_balance_terms", &AmrSystem::selected_accepted_balance_terms, + py::arg("route"), py::arg("block"), py::arg("component"), py::arg("levels"), + py::arg("automatic_terms")) .def("_consume_step_projections", &AmrSystem::consume_step_projections) .def("record_program_diagnostic", &AmrSystem::record_program_diagnostic, py::arg("name"), py::arg("value")) diff --git a/python/bindings/core/init/init_system.cpp b/python/bindings/core/init/init_system.cpp index cca9cc0ed..51c260046 100644 --- a/python/bindings/core/init/init_system.cpp +++ b/python/bindings/core/init/init_system.cpp @@ -355,6 +355,9 @@ void bind_system_program(py::class_& cls) { .def("program_diagnostic", &System::program_diagnostic, py::arg("name")) .def("program_diagnostics", &System::program_diagnostics) .def("_accepted_balance_terms", &System::accepted_balance_terms, py::arg("route")) + .def("_selected_accepted_balance_terms", &System::selected_accepted_balance_terms, + py::arg("route"), py::arg("block"), py::arg("component"), py::arg("levels"), + py::arg("automatic_terms")) .def("_consume_step_projections", &System::consume_step_projections) // ADC-542: the native collective reduction over a named block the diagnostics driver drives to // fire a declared typed measure (Norm / Integral / MinMax) each cadence tick, and the sink the diff --git a/python/pops/_pops.pyi b/python/pops/_pops.pyi index 4b8c5d49d..0c6ffdc46 100644 --- a/python/pops/_pops.pyi +++ b/python/pops/_pops.pyi @@ -277,6 +277,14 @@ class System: def solve_fields(self) -> _SolveReport: ... def _consume_step_projections(self) -> list[str]: ... def _accepted_balance_terms(self, route: str) -> dict[str, float]: ... + def _selected_accepted_balance_terms( + self, + route: str, + block: str, + component: int, + levels: list[int], + automatic_terms: list[str], + ) -> dict[str, float]: ... def output_state_local_pieces( self, block: str, level: int ) -> tuple[dict[str, object], ...]: ... @@ -297,6 +305,14 @@ class AmrSystem: def configured_n_levels(self) -> int: ... def _consume_step_projections(self) -> list[str]: ... def _accepted_balance_terms(self, route: str) -> dict[str, float]: ... + def _selected_accepted_balance_terms( + self, + route: str, + block: str, + component: int, + levels: list[int], + automatic_terms: list[str], + ) -> dict[str, float]: ... def materialize_program_restart_histories( self, payload: bytes, diff --git a/python/pops/runtime/_runtime_consumers.py b/python/pops/runtime/_runtime_consumers.py index a0d040ee1..872cb4466 100644 --- a/python/pops/runtime/_runtime_consumers.py +++ b/python/pops/runtime/_runtime_consumers.py @@ -2433,9 +2433,44 @@ def _validate_diagnostic_providers(self) -> None: if reductions == {"accepted_balance"}: if len(quantity.execution["operations"]) != 1: raise ValueError("accepted balance requires exactly one native evidence route") - if quantity.execution["role"] is not None: - raise ValueError("accepted balance route cannot carry a component role") - if not callable(getattr(engine, "_accepted_balance_terms", None)): + operation, = quantity.execution["operations"] + automatic_terms = tuple(operation.get("automatic_terms", ())) + if automatic_terms: + if not callable( + getattr(engine, "_selected_accepted_balance_terms", None) + ): + raise NotImplementedError( + "automatic balance terms require native " + "_selected_accepted_balance_terms(...)" + ) + component = operation["balance_component"] + if component >= len(names): + raise ValueError( + "automatic balance component %d is outside block %r width %d" + % (component, block, len(names)) + ) + if quantity.execution["role"] is not None: + role_component, _ = self._diagnostic_component( + names, roles, quantity.execution["role"] + ) + if role_component != component: + raise ValueError( + "automatic balance role selects component %d but ledger " + "declares component %d" % (role_component, component) + ) + if "reflux" in automatic_terms and not layout.adaptive: + raise NotImplementedError( + "automatic reflux balance requires an adaptive hierarchy" + ) + if ( + "projection" in automatic_terms + and layout.geometry.cell_measure != CARTESIAN_CELL_AREA + ): + raise NotImplementedError( + "automatic projection balance requires exact Cartesian cell " + "measure support" + ) + elif not callable(getattr(engine, "_accepted_balance_terms", None)): raise NotImplementedError( "balance diagnostic requires native _accepted_balance_terms(route)" ) @@ -2556,14 +2591,31 @@ def _native_diagnostic_reduction( return float(cast(Any, native)(block, kind, component)), False @staticmethod - def _native_balance_terms(engine: Any, route: str) -> Any: + def _native_balance_terms( + engine: Any, + route: str, + *, + block: str, + component: int, + levels: tuple[int, ...], + automatic_terms: tuple[str, ...], + ) -> Any: """Read one current-attempt balance tuple from the native transaction mailbox.""" from pops.output.diagnostics import BalanceTerms - native = getattr(engine, "_accepted_balance_terms", None) + native_name = ( + "_selected_accepted_balance_terms" + if automatic_terms + else "_accepted_balance_terms" + ) + native = getattr(engine, native_name, None) if not callable(native): raise RuntimeError("installed runtime has no accepted balance evidence provider") - raw = native(route) + raw = ( + native(route, block, component, list(levels), list(automatic_terms)) + if automatic_terms + else native(route) + ) required = { "storage_change", "outward_boundary_flux", @@ -2612,8 +2664,16 @@ def _diagnostic_values( if "accepted_balance" in skip_reductions: continue operation, = execution["operations"] + automatic_terms = tuple(operation.get("automatic_terms", ())) + component = operation.get("balance_component", 0) balance = self._native_balance_terms( - engine, operation["balance_route"]) + engine, + operation["balance_route"], + block=block, + component=component, + levels=levels, + automatic_terms=automatic_terms, + ) terms = { "storage_change": balance.storage_change, "outward_boundary_flux": balance.outward_boundary_flux, diff --git a/src/runtime/amr/amr_system.cpp b/src/runtime/amr/amr_system.cpp index 3e0403973..22050acba 100644 --- a/src/runtime/amr/amr_system.cpp +++ b/src/runtime/amr/amr_system.cpp @@ -3579,6 +3579,28 @@ std::map AmrSystem::accepted_balance_terms(const std::strin "transaction"); return p_->program_.accepted_balance_terms(route, "AmrSystem"); } +std::map AmrSystem::selected_accepted_balance_terms( + const std::string& route, const std::string& block, int component, + const std::vector& levels, const std::vector& automatic_terms) const { + if (!p_->external_step_transaction_active_ || p_->external_step_transaction_committed_) + throw std::runtime_error( + "AmrSystem::_selected_accepted_balance_terms requires an active uncommitted external step " + "transaction"); + if (!p_->runtime) + throw std::runtime_error( + "AmrSystem::_selected_accepted_balance_terms requires an installed AMR runtime"); + const std::size_t runtime_block = p_->block_index_or_throw(block); + if (component < 0 || component >= p_->runtime->block_n_vars(runtime_block)) + throw std::out_of_range( + "AmrSystem::_selected_accepted_balance_terms component is out of range"); + if (levels.empty() || std::any_of(levels.begin(), levels.end(), [&](int level) { + return level < 0 || level >= p_->runtime->nlev(); + })) + throw std::out_of_range( + "AmrSystem::_selected_accepted_balance_terms level is out of active hierarchy range"); + return p_->program_.selected_accepted_balance_terms( + route, static_cast(runtime_block), component, levels, automatic_terms, "AmrSystem"); +} void AmrSystem::begin_step_projection_report() { p_->program_.begin_step_projection_report(); } diff --git a/src/runtime/system/system_program.cpp b/src/runtime/system/system_program.cpp index ee3c308f2..6f1ceb023 100644 --- a/src/runtime/system/system_program.cpp +++ b/src/runtime/system/system_program.cpp @@ -435,6 +435,23 @@ std::map System::accepted_balance_terms(const std::string& ro "System::_accepted_balance_terms requires an active uncommitted external step transaction"); return p_->program_.accepted_balance_terms(route, "System"); } +std::map System::selected_accepted_balance_terms( + const std::string& route, const std::string& block, int component, + const std::vector& levels, const std::vector& automatic_terms) const { + if (!p_->external_step_transaction_ || p_->external_step_transaction_committed_) + throw std::runtime_error( + "System::_selected_accepted_balance_terms requires an active uncommitted external step " + "transaction"); + const int runtime_block = p_->index(block); + const auto& state = p_->find(block); + if (component < 0 || component >= state.ncomp) + throw std::out_of_range("System::_selected_accepted_balance_terms component is out of range"); + if (levels != std::vector{0}) + throw std::invalid_argument( + "System::_selected_accepted_balance_terms requires exactly uniform level 0"); + return p_->program_.selected_accepted_balance_terms(route, runtime_block, component, levels, + automatic_terms, "System"); +} void System::begin_step_projection_report() { p_->program_.begin_step_projection_report(); } From 7a57c4d6c068f90e7c9d928a8361a305dc2dd5a3 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:50:24 +0200 Subject: [PATCH 098/109] test(balance): prove qualified native term selection (ADC-686) --- .../runtime/test_program_runtime.cpp | 31 ++++++++ ...test_automatic_projection_balance_fence.py | 6 +- .../test_automatic_reflux_balance_fence.py | 8 ++- ...qualified_automatic_balance_route_fence.py | 72 +++++++++++++++++++ ...est_async_scientific_output_diagnostics.py | 38 ++++++++++ .../unit/runtime/test_consumer_authoring.py | 39 ++++++++++ .../unit/runtime/test_diagnostics_typed.py | 33 +++++++++ .../python/unit/time/test_time_ops_polish.py | 62 +++++++++++++++- 8 files changed, 285 insertions(+), 4 deletions(-) create mode 100644 tests/python/architecture/test_qualified_automatic_balance_route_fence.py diff --git a/tests/cpp/integration/runtime/test_program_runtime.cpp b/tests/cpp/integration/runtime/test_program_runtime.cpp index 7dbdd6202..a70d1481f 100644 --- a/tests/cpp/integration/runtime/test_program_runtime.cpp +++ b/tests/cpp/integration/runtime/test_program_runtime.cpp @@ -164,6 +164,37 @@ TEST(ProgramRuntime, AutomaticBalanceDueMarkerIsAttemptLocalMonotoneAndReplaySaf EXPECT_FALSE(state.automatic_balance_capture_due()); } +TEST(ProgramRuntime, SelectedAutomaticBalanceTermsRequireCompleteQualifiedEvidence) { + runtime::program::ProgramRuntimeState state; + const std::string route = "pops.balance-ledger-route.v1:sha256:" + std::string(64, '5'); + state.begin_step_projection_report(); + state.run_balance_due_window(0, "test", [&] { + state.note_automatic_balance_capture_due(true, "test"); + state.record_balance_term(route, "storage_change", 1.0, "test"); + state.record_balance_term(route, "outward_boundary_flux", 2.0, "test"); + state.record_balance_term(route, "sources", 3.0, "test"); + state.record_automatic_balance_term(2, 0, 1, "projection", 0.25, "test"); + state.record_automatic_balance_term(2, 1, 1, "projection", 0.75, "test"); + state.record_automatic_balance_term(2, 0, 1, "reflux", 0.5, "test"); + }); + state.complete_balance_step(true); + + const auto selected = + state.selected_accepted_balance_terms(route, 2, 1, {0, 1}, {"projection", "reflux"}, "test"); + EXPECT_EQ(selected.at("storage_change"), 1.0); + EXPECT_EQ(selected.at("outward_boundary_flux"), 2.0); + EXPECT_EQ(selected.at("sources"), 3.0); + EXPECT_EQ(selected.at("projection"), 1.0); + EXPECT_EQ(selected.at("reflux"), 0.5); + + EXPECT_THROW((void)state.selected_accepted_balance_terms(route, 2, 1, {0, 1, 2}, + {"projection", "reflux"}, "test"), + std::runtime_error); + EXPECT_THROW((void)state.selected_accepted_balance_terms(route, 2, 1, {0, 2}, + {"projection", "reflux"}, "test"), + std::invalid_argument); +} + TEST(ProgramRuntime, SelectiveReplayCompilesBalanceOffAndRestoresTheGuard) { runtime::program::ProgramRuntimeState state; const std::string contract = "pops.balance-due-contract.v1:sha256:" + std::string(64, '3'); diff --git a/tests/python/architecture/test_automatic_projection_balance_fence.py b/tests/python/architecture/test_automatic_projection_balance_fence.py index 8ce73ec53..14064e070 100644 --- a/tests/python/architecture/test_automatic_projection_balance_fence.py +++ b/tests/python/architecture/test_automatic_projection_balance_fence.py @@ -79,7 +79,11 @@ def test_projection_delta_is_captured_only_when_due_and_stays_qualified() -> Non "std::map accepted_balance_terms(", "void begin_balance_due_window(", ) - assert "automatic_balance_terms_" not in accepted + explicit_only = accepted.split( + "std::map selected_accepted_balance_terms(", 1 + )[0] + assert "automatic_balance_terms_" not in explicit_only + assert "automatic_balance_terms_" in accepted def test_uniform_projection_evidence_uses_exact_available_measure() -> None: diff --git a/tests/python/architecture/test_automatic_reflux_balance_fence.py b/tests/python/architecture/test_automatic_reflux_balance_fence.py index 7b2c866a6..2b300d152 100644 --- a/tests/python/architecture/test_automatic_reflux_balance_fence.py +++ b/tests/python/architecture/test_automatic_reflux_balance_fence.py @@ -40,8 +40,12 @@ def test_automatic_balance_mailbox_is_attempt_local_and_not_a_route_fallback() - "std::map accepted_balance_terms(", "void begin_balance_due_window(", ) - assert "step_balance_terms_" in accepted - assert "automatic_balance_terms_" not in accepted + explicit_only = accepted.split( + "std::map selected_accepted_balance_terms(", 1 + )[0] + assert "step_balance_terms_" in explicit_only + assert "automatic_balance_terms_" not in explicit_only + assert "automatic_balance_terms_" in accepted uniform = UNIFORM_IMPL.read_text() adaptive = AMR_IMPL.read_text() diff --git a/tests/python/architecture/test_qualified_automatic_balance_route_fence.py b/tests/python/architecture/test_qualified_automatic_balance_route_fence.py new file mode 100644 index 000000000..71768e4c8 --- /dev/null +++ b/tests/python/architecture/test_qualified_automatic_balance_route_fence.py @@ -0,0 +1,72 @@ +"""ADC-686: public Balance routes select qualified native evidence fail-closed.""" + +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +LEDGER = ROOT / "python" / "pops" / "_balance_contract.py" +MEASURES = ROOT / "python" / "pops" / "diagnostics" / "measures.py" +CONSUMERS = ROOT / "python" / "pops" / "runtime" / "_runtime_consumers.py" +PROGRAM_STATE = ( + ROOT / "include" / "pops" / "runtime" / "program" / "program_runtime_state.hpp" +) +SYSTEM = ROOT / "src" / "runtime" / "system" / "system_program.cpp" +AMR = ROOT / "src" / "runtime" / "amr" / "amr_system.cpp" +SYSTEM_BINDING = ROOT / "python" / "bindings" / "core" / "init" / "init_system.cpp" +AMR_BINDING = ROOT / "python" / "bindings" / "core" / "init" / "init_amr.cpp" + + +def _between(text: str, begin: str, end: str) -> str: + return text.split(begin, 1)[1].split(end, 1)[0] + + +def test_public_ledger_owns_role_and_exact_automatic_term_selection() -> None: + ledger = LEDGER.read_text() + assert "role: Any = None" in ledger + assert "component: int | None = None" in ledger + assert "automatic_terms: tuple[str, ...] = ()" in ledger + assert '{"reflux", "projection"}' in ledger + + measures = MEASURES.read_text() + balance = _between(measures, "class Balance(_Measure):", "class ConservationCheck") + assert "role=ledger.role" in balance + assert '"automatic_terms": list(self.ledger.automatic_terms)' in balance + assert '"balance_component": self.ledger.component' in balance + + +def test_runtime_uses_selected_native_entrypoint_only_for_delegated_terms() -> None: + consumers = CONSUMERS.read_text() + native = _between( + consumers, + "def _native_balance_terms(", + "def _diagnostic_values(", + ) + assert '"_selected_accepted_balance_terms"' in native + assert 'if automatic_terms' in native + assert "native(route, block, component, list(levels), list(automatic_terms))" in native + assert "else native(route)" in native + + for binding in (SYSTEM_BINDING, AMR_BINDING): + assert '"_selected_accepted_balance_terms"' in binding.read_text() + + +def test_native_selector_requires_complete_owner_level_component_evidence() -> None: + state = PROGRAM_STATE.read_text() + selector = _between( + state, + "std::map selected_accepted_balance_terms(", + "void begin_balance_due_window(", + ) + assert "AutomaticBalanceKey key{runtime_block, levels[index], component, term}" in selector + assert "native producer omitted term" in selector + assert "both Program and native producer authority" in selector + assert 'term == "reflux" ? levels.size() - 1 : levels.size()' in selector + + uniform = SYSTEM.read_text() + assert "const int runtime_block = p_->index(block);" in uniform + assert "levels != std::vector{0}" in uniform + + adaptive = AMR.read_text() + assert "const std::size_t runtime_block = p_->block_index_or_throw(block);" in adaptive + assert "p_->runtime->block_n_vars(runtime_block)" in adaptive + assert "p_->runtime->nlev()" in adaptive diff --git a/tests/python/unit/output/test_async_scientific_output_diagnostics.py b/tests/python/unit/output/test_async_scientific_output_diagnostics.py index 6a1298553..7fb6b856a 100644 --- a/tests/python/unit/output/test_async_scientific_output_diagnostics.py +++ b/tests/python/unit/output/test_async_scientific_output_diagnostics.py @@ -30,6 +30,7 @@ from pops.output._restart_provider import RestartAuthority from pops.output._writers.common import writer_session_authority from pops.problem.handles import BlockHandle +from pops.runtime._runtime_consumers import RuntimeConsumerPublisher from pops.runtime._runtime_instance import RuntimeInstance from pops.time import Clock, every from tests.python.support.layout_plan import cartesian_grid @@ -282,6 +283,43 @@ def _accepted_balance_terms(self, route): } +def test_selected_native_balance_forwards_exact_owner_coordinates(): + class _SelectedExecutor: + def __init__(self): + self.call = None + + def _selected_accepted_balance_terms( + self, route, block, component, levels, automatic_terms + ): + self.call = (route, block, component, levels, automatic_terms) + return { + "storage_change": 7.0, + "outward_boundary_flux": 2.0, + "sources": 3.0, + "reflux": 1.0, + "projection": 0.5, + } + + executor = _SelectedExecutor() + terms = RuntimeConsumerPublisher._native_balance_terms( + executor, + "route", + block="fluid", + component=2, + levels=(0, 1), + automatic_terms=("projection", "reflux"), + ) + + assert executor.call == ( + "route", + "fluid", + 2, + [0, 1], + ["projection", "reflux"], + ) + assert terms.residual == pytest.approx(4.5) + + def _async_balance_runtime(tmp_path: Path): base = _install() mode = _scientific_output_mode(base.artifact) diff --git a/tests/python/unit/runtime/test_consumer_authoring.py b/tests/python/unit/runtime/test_consumer_authoring.py index c5d2ed660..73569e27f 100644 --- a/tests/python/unit/runtime/test_consumer_authoring.py +++ b/tests/python/unit/runtime/test_consumer_authoring.py @@ -293,6 +293,45 @@ def test_balance_consumer_resolves_one_exact_native_ledger_route(): assert contract.identity.domain == "balance-due-contract" +def test_balance_consumer_retains_native_term_selector_in_due_contract(): + case, block, state = _case() + clock = Clock("macro", owner=case.owner_path) + schedule = every(4, clock=clock) + ledger = BalanceLedger( + "mass-native", automatic_terms=("projection", "reflux") + ) + graph = ConsumerGraph.from_consumers(( + ScientificOutput( + format=ParaView(), + schedule=schedule, + fields=(state,), + diagnostics=(Balance(ledger, block=block),), + target="state/native-balance", + ), + )) + case.consumers(graph) + pops.validate(case) + subjects = case.layout_subjects() + layout = normalize_layout_plan( + Uniform(cartesian_grid(n=8)), + owner=case.owner_path.canonical(), + states=subjects.states, + fields=subjects.fields, + blocks=subjects.blocks, + handle_resolver=case.resolve, + ) + + resolved = graph.resolve(case.resolve, layout, owner=case.owner_path.canonical()) + quantity, = resolved.nodes[0].diagnostic_quantities + operation, = quantity.execution["operations"] + route = ledger.route_identity(case.resolve(block)) + contract = BalanceDueContract.from_consumer_graph(resolved) + + assert operation["automatic_terms"] == ("projection", "reflux") + assert operation["balance_component"] == 0 + assert contract.route(route.token).automatic_terms == ("projection", "reflux") + + def test_balance_consumer_refuses_a_schedule_that_can_fire_at_start(): case, block, state = _case() clock = Clock("macro", owner=case.owner_path) diff --git a/tests/python/unit/runtime/test_diagnostics_typed.py b/tests/python/unit/runtime/test_diagnostics_typed.py index a2670174d..b93c3ea36 100644 --- a/tests/python/unit/runtime/test_diagnostics_typed.py +++ b/tests/python/unit/runtime/test_diagnostics_typed.py @@ -126,6 +126,39 @@ def test_balance_uses_one_typed_native_attempt_route(): ConservationCheck(balance).diagnostic_execution() +def test_balance_ledger_selects_exact_native_component_terms(): + ledger = BalanceLedger( + "mass-native", + role=Density(), + automatic_terms=("projection", "reflux"), + ) + balance = Balance(ledger, block=_NE_BLOCK) + execution = balance.diagnostic_execution() + operation, = execution["operations"] + + assert execution["role"] == "Density" + assert operation["automatic_terms"] == ["projection", "reflux"] + assert operation["balance_component"] == 0 + assert balance.options()["role"] == "Density" + assert ledger.to_data()["role"] == "Density" + assert ledger.to_data()["component"] == 0 + assert ledger.to_data()["automatic_terms"] == ["projection", "reflux"] + assert ledger.identity != BalanceLedger("mass-native").identity + + with pytest.raises(TypeError, match="ComponentRole"): + BalanceLedger("bad-role", role="Density") + reordered = BalanceLedger( + "canonical-order", automatic_terms=("reflux", "projection") + ) + assert reordered.automatic_terms == ("projection", "reflux") + with pytest.raises(ValueError, match="must be unique"): + BalanceLedger("duplicate", automatic_terms=("reflux", "reflux")) + with pytest.raises(ValueError, match="only reflux and projection"): + BalanceLedger("bad-producer", automatic_terms=("sources",)) + with pytest.raises(TypeError, match="non-negative int"): + BalanceLedger("bad-component", component=-1, automatic_terms=("projection",)) + + # --- Integral / MinMax ------------------------------------------------------------------ def test_integral_is_a_sum_reduction(): mass = Integral(role=Density()) diff --git a/tests/python/unit/time/test_time_ops_polish.py b/tests/python/unit/time/test_time_ops_polish.py index b996b6950..788566df8 100644 --- a/tests/python/unit/time/test_time_ops_polish.py +++ b/tests/python/unit/time/test_time_ops_polish.py @@ -59,7 +59,7 @@ def t(): return time -def _balance_due_contract(route, *schedules): +def _balance_due_contract(route, *schedules, automatic_terms=()): return BalanceDueContract( make_identity("consumer-graph", {"test": "balance-due"}), ( @@ -72,6 +72,7 @@ def _balance_due_contract(route, *schedules): ) for index, schedule in enumerate(schedules) ), + automatic_terms, ), ), ) @@ -378,6 +379,65 @@ def test_record_balance_emits_exact_five_term_native_attempt_mailbox(t): assert "ctx.note_automatic_balance_capture_due(" not in unreachable_source +def test_record_balance_delegates_selected_native_terms_without_placeholders(t): + from pops.diagnostics import BalanceLedger + + P = t.Program("native-balance-terms") + U = typed_state(P, "blk") + total = P.sum(U) + ledger = BalanceLedger( + "mass-native", automatic_terms=("projection", "reflux") + ) + records = P.record_balance( + ledger, + storage_change=total, + outward_boundary_flux=total * 2.0, + sources=total * 3.0, + ) + route = ledger.route_identity(U.block) + assert tuple(record.attrs["term"] for record in records) == ( + "storage_change", + "outward_boundary_flux", + "sources", + ) + endpoint = typed_state(P, "blk", state_name="U").next + P.commit(endpoint, P.value("balance_next", U, at=endpoint.point)) + contract = _balance_due_contract( + route, + every(2, clock=P.clock), + automatic_terms=("projection", "reflux"), + ) + source = emit_cpp_program(P, balance_due_contract=contract) + assert source.count("ctx.record_balance_term(") == 3 + assert source.count("ctx.note_automatic_balance_capture_due(") == 1 + + P_bad = t.Program("duplicate-native-balance-term") + U_bad = typed_state(P_bad, "blk") + total_bad = P_bad.sum(U_bad) + with pytest.raises(ValueError, match="owned by.*native automatic producer"): + P_bad.record_balance( + ledger, + storage_change=total_bad, + outward_boundary_flux=total_bad, + sources=total_bad, + projection=total_bad, + ) + + component_ledger = BalanceLedger( + "component-one", + component=1, + automatic_terms=("projection",), + ) + with pytest.raises(ValueError, match="selects components.*component 1"): + P_bad.record_balance( + component_ledger, + storage_change=total_bad, + outward_boundary_flux=total_bad, + sources=total_bad, + reflux=total_bad, + ) + + def test_balance_due_contract_unions_consumers_and_ignores_static_false(t): from pops.diagnostics import BalanceLedger From e7e70f66b97466fc3af661a33df86623e6635e95 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sat, 1 Aug 2026 23:50:33 +0200 Subject: [PATCH 099/109] docs(balance): define automatic ledger authority (ADC-686) --- docs/design/exact-output-consumers.md | 72 +++++++++++++++++++-------- 1 file changed, 50 insertions(+), 22 deletions(-) diff --git a/docs/design/exact-output-consumers.md b/docs/design/exact-output-consumers.md index 45e587f90..6f74b73d8 100644 --- a/docs/design/exact-output-consumers.md +++ b/docs/design/exact-output-consumers.md @@ -53,6 +53,28 @@ ScientificOutput( ) ``` +When the native runtime owns an AMR reflux correction and/or an authored projection, the ledger can +delegate those exact terms instead of requiring zero placeholders. `component` is the exact +conservative index shared by the explicit Program sums and native evidence (it defaults to zero for +a scalar state); the optional typed role is checked against that index at bind: + +```python +from pops.physics.roles import Density + +mass = BalanceLedger( + "mass", + role=Density(), + component=0, + automatic_terms=("projection", "reflux"), +) +program.record_balance( + mass, + storage_change=storage_increment, + outward_boundary_flux=boundary_flux_increment, + sources=source_increment, +) +``` + Le fournisseur possède l'extension. Une cible comme `solution/tracer.vtu` est refusée dès l'authoring, avant le bind ; elle empêcherait le changement de format et entrerait en collision au deuxième échantillon. Chaque pas accepté dû publie immédiatement un fichier distinct sous le chemin @@ -451,15 +473,20 @@ schedule and transaction. Its reductions are completed on the simulation thread the post-commit worker receives only immutable arrays and scalar payloads, never the native mailbox or communicator facade. -Each argument to `record_balance` is a signed, time-integrated native Program sum/dot reduction, -or scalar arithmetic composed only from such reductions and exact literals. +Each non-automatic argument to `record_balance` is a signed, time-integrated native Program sum/dot +reduction, or scalar arithmetic composed only from such reductions and exact literals. When any +term is delegated to a native producer, every explicit term must instead be composed from +component-qualified `sum` reductions for the ledger's exact `component`; an all-state dot product +cannot be reconciled with one component's reflux/projection evidence. The reported residual is `storage_change + outward_boundary_flux - sources - reflux - projection`. The native attempt mailbox accumulates repeated cadence/substep invocations, rejects missing or non-finite terms, and is cleared before the next attempt. The consumer reads it only while the outer accepted-step transaction still retains the pre-step image. Python therefore packages the five returned scalars and residual but never traverses arrays, invents a zero term, or reuses a -previous step. A rejected attempt or failed consumer publication restores the mailbox with the -rest of the native transaction. +previous step. Selected automatic terms are resolved by exact runtime block, active hierarchy level +and conservative component. A missing coordinate, a non-finite value, or simultaneous Program and +native authority for one term fails the accepted transaction. A rejected attempt or failed consumer +publication restores both mailboxes with the rest of the native transaction. The `pops.balance-term` namespace is reserved. Ordinary `Program.record_scalar(...)` authoring and the Python runtime diagnostic binding both reject it; generated `record_balance` code reaches a @@ -476,8 +503,9 @@ by an OR of their exact accepted-step periods. `Always` and `when(True)` are per The compiler traces the complete reduction/scalar chain rather than scheduling only the terminal records. If a value is also consumed by an ordinary Program diagnostic or another non-balance operation, that shared producer remains unconditional so cadence fusion cannot change unrelated -semantics. A `Balance` consumer with no matching five-term `Program.record_balance` producer fails -before native code generation. Program stride/substeps use one attempt-local outer accepted-step +semantics. A `Balance` consumer with no complete matching `Program.record_balance` producer for all +non-automatic terms fails before native code generation. Program stride/substeps use one +attempt-local outer accepted-step target, so every substep of one due public step sees the same decision and accumulates into the same attempt mailbox. The cadence is authored once as part of the Program identity, for example `program.cadence(substeps=2, stride=3)`, then authenticated and installed before runtime freeze on @@ -498,22 +526,22 @@ balance reductions are not yet skipped. This fallback can add work but cannot su evidence. A zero-step run has no accepted native occurrence: its coincident start/end moment cannot publish an accepted-step consumer, including `Balance`. -This public route still consumes explicit evidence: a Program that cannot produce every actual term -cannot declare `Balance`. Native operator instrumentation is deliberately kept in a separate, -qualified attempt-local mailbox until a resolved quantity selector can prove which -`BalanceLedger` route owns each block/level/component contribution. Generated code publishes the OR -of the exact due route decisions before the first Program operator; the marker is monotone for the -attempt, disabled during replay, and reset at attempt entry. Consequently off-cadence steps do not -pay for automatic operator reductions. - -That private mailbox currently captures the signed AMR reflux correction and the before/after -projection delta. Uniform Cartesian projection uses the authenticated cell measure and embedded -boundary mask; AMR projection excludes covered coarse cells and performs one component-vector -collective per participating level. Polar projection stays absent because no exact per-cell polar -volume provider exists on this path. Automatic physical-boundary flux and source evidence are also -not yet producers. None of these private values is read by `accepted_balance_terms()`, so this -instrumentation does not silently complete an authored five-term balance or widen the public -contract. +The selected public route now consumes signed AMR reflux corrections and before/after projection +deltas from the separate qualified attempt mailbox. Uniform Cartesian projection uses the +authenticated cell measure and embedded-boundary mask; AMR projection excludes covered coarse cells +and performs one component-vector collective per participating level. A reflux selection requires +an adaptive hierarchy and expects one contribution for every active parent/fine interface; +projection expects one for every selected active level. Generated code publishes the OR of the exact +due route decisions before the first Program operator; the marker is monotone for the attempt, +disabled during replay, and reset at attempt entry. Consequently off-cadence steps do not pay for +automatic operator reductions. + +The capability remains deliberately bounded. Polar projection is rejected because no exact +per-cell polar volume provider exists on this path. Automatic physical-boundary flux and source +evidence are not yet producers and therefore remain explicit `Program.record_balance` arguments. +The native selector never substitutes a missing automatic value with zero (except the exact reflux +identity for a hierarchy with no coarse/fine interface), and the legacy all-explicit ledger route +retains its original identity and behavior. Checkpoint remains a separate restart effect. These consumers do not define a checkpoint schema or reader and do not call the scientific-output manifest a restart identity. The checkpoint provider From 8f2376997daba98c33c324a8bb89801a327a965e Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 01:08:21 +0200 Subject: [PATCH 100/109] release: bind installed public API distribution identity --- scripts/prove_public_api_parity.py | 87 +++++++++++++++++-- .../test_public_api_parity_proof.py | 42 +++++++-- 2 files changed, 119 insertions(+), 10 deletions(-) diff --git a/scripts/prove_public_api_parity.py b/scripts/prove_public_api_parity.py index f09efe785..256b02c4b 100644 --- a/scripts/prove_public_api_parity.py +++ b/scripts/prove_public_api_parity.py @@ -5,6 +5,8 @@ import argparse from collections.abc import Mapping, Sequence +from email import policy +from email.parser import BytesParser import hashlib import importlib.metadata import json @@ -18,7 +20,7 @@ ROOT = Path(__file__).resolve().parents[1] SOURCE_PACKAGE = ROOT / "python" / "pops" -PROOF_SCHEMA_VERSION = 2 +PROOF_SCHEMA_VERSION = 3 TYPED_PAYLOAD_SUFFIXES = (".py", ".pyi") PUBLIC_ROOT = ( "Model", @@ -149,6 +151,7 @@ def _symbol(name): snapshot = { "public": public, "symbols": {name: _symbol(name) for name in public}, + "package_version": pops.__version__, "case_is_explicit_type": True, "qualified_handles": True, "pure_authoring": True, @@ -211,6 +214,38 @@ def _wheel_manifest(archive: zipfile.ZipFile) -> dict[str, str]: return manifest +def _distribution_identity(payload: bytes, *, label: str) -> dict[str, str]: + try: + metadata = BytesParser(policy=policy.default).parsebytes(payload) + except (TypeError, ValueError) as exc: + raise PublicApiParityError("%s distribution METADATA is unreadable" % label) from exc + name = metadata.get("Name") + version = metadata.get("Version") + if not isinstance(name, str) or not name.strip() \ + or not isinstance(version, str) or not version.strip(): + raise PublicApiParityError( + "%s distribution METADATA has no exact Name/Version" % label) + normalized = name.strip().lower().replace("_", "-").replace(".", "-") + if normalized != "pops": + raise PublicApiParityError("%s distribution name is not PoPS" % label) + return { + "name": name.strip(), + "version": version.strip(), + "metadata_sha256": _sha256_bytes(payload), + } + + +def _wheel_distribution_identity(archive: zipfile.ZipFile) -> dict[str, str]: + names = [info.filename for info in archive.infolist() if not info.is_dir()] + if len(names) != len(set(names)): + raise PublicApiParityError("release wheel contains duplicate members") + metadata_names = [name for name in names if name.endswith(".dist-info/METADATA")] + if len(metadata_names) != 1: + raise PublicApiParityError("release wheel has no unique distribution METADATA") + return _distribution_identity( + archive.read(metadata_names[0]), label="wheel") + + def _safe_extract(archive: zipfile.ZipFile, destination: Path) -> None: for info in archive.infolist(): relative = PurePosixPath(info.filename) @@ -274,7 +309,7 @@ def _require_manifest_parity( ) -def _installed_package_from_distribution() -> Path: +def _installed_package_from_distribution() -> tuple[Path, dict[str, str]]: try: distribution = importlib.metadata.distribution("PoPS") except importlib.metadata.PackageNotFoundError as exc: @@ -282,6 +317,18 @@ def _installed_package_from_distribution() -> Path: files = distribution.files if files is None: raise PublicApiParityError("the installed PoPS distribution has no file inventory") + metadata_files = [ + row + for row in files + if PurePosixPath(str(row)).as_posix().endswith(".dist-info/METADATA") + ] + if len(metadata_files) != 1: + raise PublicApiParityError( + "the installed PoPS distribution has no unique METADATA") + metadata_path = Path(distribution.locate_file(metadata_files[0])).resolve() + if not metadata_path.is_file(): + raise PublicApiParityError("the installed PoPS distribution METADATA is absent") + identity = _distribution_identity(metadata_path.read_bytes(), label="installed") package_initializers = [ row for row in files if PurePosixPath(str(row)).as_posix() == "pops/__init__.py" ] @@ -294,7 +341,7 @@ def _installed_package_from_distribution() -> Path: try: package.relative_to(ROOT) except ValueError: - return package + return package, identity raise PublicApiParityError( "the installed-package proof resolved inside the source checkout: %s" % package) @@ -303,6 +350,7 @@ def build_proof( wheel: Path, *, installed_package: Path | None = None, + installed_distribution: Mapping[str, str] | None = None, ) -> dict[str, Any]: """Compare one exact wheel archive with the current source checkout.""" retained = wheel.expanduser().resolve() @@ -310,6 +358,9 @@ def build_proof( raise PublicApiParityError("release artifact is not one readable wheel") source_manifest = _typed_manifest(SOURCE_PACKAGE, label="source") installed = None if installed_package is None else installed_package.expanduser().resolve() + if installed_distribution is not None and installed is None: + raise PublicApiParityError( + "installed distribution identity requires an installed package") if installed is not None: try: installed.relative_to(ROOT) @@ -322,6 +373,7 @@ def build_proof( with tempfile.TemporaryDirectory(prefix="pops-public-api-") as temporary: extracted = Path(temporary) with zipfile.ZipFile(retained) as archive: + wheel_distribution = _wheel_distribution_identity(archive) wheel_manifest = _wheel_manifest(archive) _require_manifest_parity( source_manifest, wheel_manifest, label="wheel") @@ -337,14 +389,29 @@ def build_proof( raise PublicApiParityError("release wheel is unreadable: %s" % exc) from exc if wheel_snapshot != source_snapshot: raise PublicApiParityError("wheel and source public API snapshots differ") + if source_snapshot.get("package_version") != wheel_distribution["version"]: + raise PublicApiParityError( + "source public API version differs from wheel distribution METADATA") if installed is not None and installed_snapshot != source_snapshot: raise PublicApiParityError("installed and source public API snapshots differ") + if installed_distribution is not None: + exact_installed_distribution = dict(installed_distribution) + if set(exact_installed_distribution) != {"name", "version", "metadata_sha256"}: + raise PublicApiParityError("installed distribution identity is malformed") + if exact_installed_distribution != wheel_distribution: + raise PublicApiParityError( + "installed distribution identity differs from wheel METADATA") if tuple(source_snapshot["public"]) != PUBLIC_ROOT: raise PublicApiParityError("public API snapshot differs from the final root contract") proof = { "schema_version": PROOF_SCHEMA_VERSION, + "producer": { + "script": "scripts/prove_public_api_parity.py", + "sha256": _sha256(Path(__file__).resolve()), + }, "wheel_path": str(retained), "wheel_sha256": _sha256(retained), + "distribution": wheel_distribution, "typed_payload_files": len(source_manifest), "typed_payload_sha256": _canonical_sha256(source_manifest), "public_api_sha256": _canonical_sha256(source_snapshot), @@ -353,6 +420,9 @@ def build_proof( "qualified_handles": source_snapshot["qualified_handles"], "py_typed": source_snapshot["py_typed"], "installed": installed is not None, + "installed_distribution": ( + None if installed_distribution is None else dict(installed_distribution) + ), } if installed is not None: proof.update({ @@ -394,8 +464,15 @@ def main(argv: Sequence[str] | None = None) -> int: parser.add_argument("--evidence", type=Path) args = parser.parse_args(argv) try: - installed = _installed_package_from_distribution() if args.installed else None - proof = build_proof(args.wheel, installed_package=installed) + if args.installed: + installed, installed_distribution = _installed_package_from_distribution() + else: + installed, installed_distribution = None, None + proof = build_proof( + args.wheel, + installed_package=installed, + installed_distribution=installed_distribution, + ) if args.evidence is not None: _write_evidence(args.evidence, proof) except (PublicApiParityError, OSError, ValueError) as exc: diff --git a/tests/python/architecture/test_public_api_parity_proof.py b/tests/python/architecture/test_public_api_parity_proof.py index cda32be7d..f352cdfba 100644 --- a/tests/python/architecture/test_public_api_parity_proof.py +++ b/tests/python/architecture/test_public_api_parity_proof.py @@ -29,6 +29,12 @@ def _load(): proof = _load() +_METADATA = "Metadata-Version: 2.3\nName: PoPS\nVersion: 1.0.0\n" + + +def _distribution_identity() -> dict[str, str]: + return proof._distribution_identity(_METADATA.encode("utf-8"), label="test") + def _synthetic_wheel(path: Path, *, omit: str | None = None) -> None: with zipfile.ZipFile(path, "w") as archive: @@ -41,7 +47,7 @@ def _synthetic_wheel(path: Path, *, omit: str | None = None) -> None: archive.write(source, "pops/" + relative) archive.writestr( "pops-1.0.0.dist-info/METADATA", - "Metadata-Version: 2.3\nName: PoPS\nVersion: 1.0.0\n", + _METADATA, ) @@ -60,11 +66,13 @@ def _installed_distribution(root: Path) -> Path: distribution = package.parent / "pops-1.0.0.dist-info" distribution.mkdir() (distribution / "METADATA").write_text( - "Metadata-Version: 2.3\nName: PoPS\nVersion: 1.0.0\n", + _METADATA, encoding="utf-8", ) (distribution / "RECORD").write_text( - "pops/__init__.py,,\n", + "pops/__init__.py,,\n" + "pops-1.0.0.dist-info/METADATA,,\n" + "pops-1.0.0.dist-info/RECORD,,\n", encoding="utf-8", ) return package @@ -75,15 +83,22 @@ def test_exact_wheel_and_source_share_public_api_typing_and_lazy_authoring(tmp_p _synthetic_wheel(wheel) installed = _installed_package(tmp_path) - evidence = proof.build_proof(wheel, installed_package=installed) + evidence = proof.build_proof( + wheel, + installed_package=installed, + installed_distribution=_distribution_identity(), + ) - assert evidence["schema_version"] == 2 + assert evidence["schema_version"] == 3 + assert evidence["producer"]["script"] == "scripts/prove_public_api_parity.py" + assert evidence["distribution"] == _distribution_identity() assert evidence["public_names"] == list(proof.PUBLIC_ROOT) assert evidence["pure_authoring"] is True assert evidence["qualified_handles"] is True assert evidence["py_typed"] is True assert evidence["typed_payload_files"] > 100 assert evidence["installed"] is True + assert evidence["installed_distribution"] == evidence["distribution"] assert evidence["installed_package"] == str(installed.resolve()) assert evidence["installed_typed_payload_sha256"] == evidence["typed_payload_sha256"] assert evidence["installed_public_api_sha256"] == evidence["public_api_sha256"] @@ -113,6 +128,20 @@ def test_installed_proof_rejects_payload_drift_and_source_checkout_alias(tmp_pat proof.build_proof(wheel, installed_package=proof.SOURCE_PACKAGE) +def test_installed_proof_rejects_distribution_identity_drift(tmp_path): + wheel = tmp_path / "pops-1.0.0-py3-none-any.whl" + _synthetic_wheel(wheel) + installed = _installed_package(tmp_path) + drifted = {**_distribution_identity(), "version": "1.0.1"} + + with pytest.raises(proof.PublicApiParityError, match="distribution identity"): + proof.build_proof( + wheel, + installed_package=installed, + installed_distribution=drifted, + ) + + def test_installed_cli_resolves_distribution_after_install_without_checkout_shadowing( tmp_path, ): @@ -145,6 +174,8 @@ def test_installed_cli_resolves_distribution_after_install_without_checkout_shad assert completed.returncode == 0, completed.stdout payload = json.loads(evidence.read_text(encoding="utf-8")) assert payload["installed"] is True + assert payload["distribution"] == _distribution_identity() + assert payload["installed_distribution"] == payload["distribution"] assert payload["installed_package"] == str(installed.resolve()) assert payload["installed_typed_payload_sha256"] == payload["typed_payload_sha256"] assert payload["installed_public_api_sha256"] == payload["public_api_sha256"] @@ -160,6 +191,7 @@ def test_release_workflow_blocks_publication_on_source_wheel_api_parity(): assert '--wheel "${wheels[0]}"' in validate assert "--installed" in validate assert 'pops-final-evidence-public-api.json' in validate + assert '--public-api-evidence "$public_api_evidence"' in validate assert validate.index("scripts/run_final_gate.py") < validate.index( "scripts/prove_public_api_parity.py") assert validate.index("scripts/prove_public_api_parity.py") < validate.index( From a43338f3afb9ccba0d7078a2099187a4fa08e6d7 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 01:08:33 +0200 Subject: [PATCH 101/109] release: authenticate installed API parity evidence --- .github/workflows/release.yml | 6 +- ...TION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md | 6 +- docs/docmap.toml | 1 + scripts/release_preflight.py | 123 +++++++++++++++++- .../architecture/test_final_release_gate.py | 76 +++++++++++ .../architecture/test_release_contract.py | 5 +- 6 files changed, 208 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dac1cf5f6..d485f0d20 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -61,11 +61,12 @@ jobs: wheels=("$RUNNER_TEMP"/wheelhouse/pops-*.whl) test "${#wheels[@]}" -eq 1 evidence="$RUNNER_TEMP/pops-final-evidence.json" + public_api_evidence="$RUNNER_TEMP/pops-final-evidence-public-api.json" python scripts/run_final_gate.py --wheel "${wheels[0]}" --evidence "$evidence" python scripts/prove_public_api_parity.py \ --wheel "${wheels[0]}" \ --installed \ - --evidence "$RUNNER_TEMP/pops-final-evidence-public-api.json" + --evidence "$public_api_evidence" python - <<'PY' from pops.runtime_environment import runtime_environment_report report = runtime_environment_report() @@ -73,7 +74,8 @@ jobs: assert report["mpi_compiled"] is False, report PY python scripts/release_preflight.py \ - --release --tag "$GITHUB_REF_NAME" --installed --evidence "$evidence" + --release --tag "$GITHUB_REF_NAME" --installed --evidence "$evidence" \ + --public-api-evidence "$public_api_evidence" - name: Retain authenticated release evidence uses: actions/upload-artifact@v7 diff --git a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md index 83be821cf..daff31dfe 100644 --- a/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md +++ b/docs/design/SPECIFICATION_TECHNIQUE_FINALE_POPS_ARCHITECTURE.md @@ -1548,7 +1548,11 @@ isolés. Les trois snapshots doivent exposer la même racine publique, les même annotations, un `Case` explicite, des handles qualifiés distincts et authoring/validation/inspection sans chargement de `_pops`. Un ancien nom public, un fichier de typage absent, un chemin provenant du checkout ou une divergence source/wheel/installé bloque la -publication. +publication. La preuve authentifie aussi le `Name`, la `Version` et le digest du `METADATA` de la +distribution installée contre ceux du wheel. Enfin `release_preflight.py` reçoit cette evidence via +`--public-api-evidence` et vérifie son producteur, le SHA-256 du wheel et le chemin du package contre +le même runtime installé que l'evidence finale ; une evidence de parité issue d'un autre wheel ou +d'une autre installation ne peut donc pas être réutilisée. Une release ne peut être déclarée conforme que par `scripts/run_final_gate.py --evidence `. La commande exige un checkout propre, diff --git a/docs/docmap.toml b/docs/docmap.toml index c5663ad22..659c54816 100644 --- a/docs/docmap.toml +++ b/docs/docmap.toml @@ -85,6 +85,7 @@ depends_on = [ "python/pops/problem/problem.py", "python/pops/time/_program/api.py", "scripts/prove_public_api_parity.py", + "scripts/release_preflight.py", ".github/workflows/release.yml", "examples/final/EXEMPLE_SPEC_FINALE_ADVECTION_SCALAIRE_COMPLET.py", "examples/final/EXEMPLE_SPEC_FINALE_MULTIPHYSIQUE_CORE.py", diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index d9401c8fd..17332cb5a 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -33,6 +33,7 @@ GENERATED = ROOT / "python" / "pops" / "_generated_release_contract.py" REQUIRED_GATES = REQUIRED_RELEASE_GATES EVIDENCE_SCHEMA_VERSION = 4 +PUBLIC_API_EVIDENCE_SCHEMA_VERSION = 3 class PreflightError(RuntimeError): @@ -236,6 +237,96 @@ def _wheel_evidence(directory: Path, gates: dict[str, Any], contract: Any) -> No raise PreflightError("release wheel name/version disagrees with the release contract") +def _public_api_evidence( + path: Path, + release_evidence: dict[str, Any], + contract: Any, +) -> None: + resolved = path.expanduser().resolve() + if _inside(ROOT, resolved) or not resolved.is_file(): + raise PreflightError( + "installed public API evidence must be one file outside the checkout") + try: + payload = json.loads(resolved.read_text(encoding="utf-8")) + except (json.JSONDecodeError, OSError) as exc: + raise PreflightError("installed public API evidence is unreadable") from exc + expected = { + "schema_version", + "producer", + "wheel_path", + "wheel_sha256", + "distribution", + "typed_payload_files", + "typed_payload_sha256", + "public_api_sha256", + "public_names", + "pure_authoring", + "qualified_handles", + "py_typed", + "installed", + "installed_distribution", + "installed_package", + "installed_typed_payload_sha256", + "installed_public_api_sha256", + } + if not isinstance(payload, dict) or set(payload) != expected \ + or payload["schema_version"] != PUBLIC_API_EVIDENCE_SCHEMA_VERSION: + raise PreflightError("installed public API evidence has an unknown schema") + producer = { + "script": "scripts/prove_public_api_parity.py", + "sha256": hashlib.sha256( + (ROOT / "scripts" / "prove_public_api_parity.py").read_bytes() + ).hexdigest(), + } + if payload["producer"] != producer: + raise PreflightError("installed public API evidence has another producer") + wheel = release_evidence["gates"]["official_build"]["evidence"]["wheel"] + if payload["wheel_sha256"] != wheel["sha256"]: + raise PreflightError("installed public API evidence belongs to another wheel") + distribution = payload["distribution"] + installed_distribution = payload["installed_distribution"] + if not isinstance(distribution, dict) or set(distribution) != { + "name", "version", "metadata_sha256"}: + raise PreflightError("public API wheel distribution identity is malformed") + if installed_distribution != distribution: + raise PreflightError("installed distribution identity differs from the release wheel") + if not isinstance(distribution["name"], str) \ + or not isinstance(distribution["version"], str) \ + or distribution["name"].lower() != "pops" \ + or distribution["version"] != contract.PACKAGE_VERSION: + raise PreflightError("public API distribution identity disagrees with the release") + digests = ( + distribution["metadata_sha256"], + payload["wheel_sha256"], + payload["typed_payload_sha256"], + payload["installed_typed_payload_sha256"], + payload["public_api_sha256"], + payload["installed_public_api_sha256"], + ) + if any(not isinstance(value, str) or re.fullmatch(r"[0-9a-f]{64}", value) is None + for value in digests): + raise PreflightError("installed public API evidence contains an invalid digest") + if payload["installed_typed_payload_sha256"] != payload["typed_payload_sha256"] \ + or payload["installed_public_api_sha256"] != payload["public_api_sha256"]: + raise PreflightError("installed public API or typing digest differs from source") + if payload["installed"] is not True or payload["pure_authoring"] is not True \ + or payload["qualified_handles"] is not True or payload["py_typed"] is not True: + raise PreflightError("installed public API evidence did not prove the final contract") + if not isinstance(payload["typed_payload_files"], int) \ + or payload["typed_payload_files"] <= 0 \ + or not isinstance(payload["public_names"], list) \ + or not payload["public_names"] \ + or not all(isinstance(name, str) and name for name in payload["public_names"]): + raise PreflightError("installed public API evidence has an empty public surface") + if not isinstance(payload["installed_package"], str): + raise PreflightError("installed public API package path is malformed") + installed_package = Path(payload["installed_package"]).resolve() + runtime_package = Path(release_evidence["runtime"]["pops_file"]).resolve().parent + if installed_package != runtime_package: + raise PreflightError( + "public API parity was not proven on the authenticated installed runtime") + + def _examples_evidence(directory: Path, gates: dict[str, Any]) -> None: examples = gates["examples"]["evidence"] reopen = gates["artifact_reopen"]["evidence"] @@ -297,7 +388,12 @@ def _examples_evidence(directory: Path, gates: dict[str, Any]) -> None: raise PreflightError("release evidence restart proof markers drifted for %s" % key) -def _evidence(path: Path, contract: Any, commit: str, runtime: dict[str, str]) -> None: +def _evidence( + path: Path, + contract: Any, + commit: str, + runtime: dict[str, str], +) -> dict[str, Any]: payload = json.loads(path.read_text(encoding="utf-8")) expected = {"schema_version", "producer", "commit_sha", "package_version", "contract_sha256", "artifact_directory", "runtime", "gates"} @@ -361,6 +457,7 @@ def _evidence(path: Path, contract: Any, commit: str, runtime: dict[str, str]) - if gates["python_conformance"]["evidence"]["selection"] != PYTHON_REQUIRED_SELECTION: raise PreflightError("release evidence Python required-lane selection drifted") _examples_evidence(directory, gates) + return payload def main() -> int: @@ -369,10 +466,18 @@ def main() -> int: parser.add_argument("--tag") parser.add_argument("--installed", action="store_true") parser.add_argument("--evidence", type=Path) + parser.add_argument("--public-api-evidence", type=Path) args = parser.parse_args() try: - if args.release and (not args.tag or not args.installed or args.evidence is None): - raise PreflightError("--release requires --tag, --installed and --evidence") + if args.release and ( + not args.tag + or not args.installed + or args.evidence is None + or args.public_api_evidence is None + ): + raise PreflightError( + "--release requires --tag, --installed, --evidence and " + "--public-api-evidence") contract = _generated() checks = _static_contract(contract) if args.release: @@ -381,8 +486,16 @@ def main() -> int: if _run("git", "status", "--porcelain"): raise PreflightError("release checkout is dirty") runtime = _installed_contract(contract) - _evidence(args.evidence, contract, commit, runtime) - checks.extend(("tag", "changelog", "installed", "evidence", "clean")) + release_evidence = _evidence(args.evidence, contract, commit, runtime) + _public_api_evidence(args.public_api_evidence, release_evidence, contract) + checks.extend(( + "tag", + "changelog", + "installed", + "evidence", + "public_api_parity", + "clean", + )) elif args.tag: _tag_contract(contract.PACKAGE_VERSION, args.tag) checks.extend(("tag", "changelog")) diff --git a/tests/python/architecture/test_final_release_gate.py b/tests/python/architecture/test_final_release_gate.py index afb28b249..ba7aba224 100644 --- a/tests/python/architecture/test_final_release_gate.py +++ b/tests/python/architecture/test_final_release_gate.py @@ -1,7 +1,9 @@ """Source-only contract checks for the final release gate (ADC-695).""" from __future__ import annotations +import hashlib import importlib.util +import json from pathlib import Path import sys import zipfile @@ -191,6 +193,80 @@ def test_release_evidence_authenticates_the_exact_retained_wheel(tmp_path): preflight._wheel_evidence(tmp_path, gates, release) +def _write_public_api_evidence(tmp_path: Path) -> tuple[Path, dict, object]: + package = tmp_path / "site-packages" / "pops" + wheel_sha256 = "a" * 64 + typed_sha256 = "b" * 64 + public_sha256 = "c" * 64 + metadata_sha256 = "d" * 64 + payload = { + "schema_version": preflight.PUBLIC_API_EVIDENCE_SCHEMA_VERSION, + "producer": { + "script": "scripts/prove_public_api_parity.py", + "sha256": hashlib.sha256( + (SCRIPTS / "prove_public_api_parity.py").read_bytes() + ).hexdigest(), + }, + "wheel_path": str(tmp_path / "pops.whl"), + "wheel_sha256": wheel_sha256, + "distribution": { + "name": "PoPS", + "version": "1.0.0", + "metadata_sha256": metadata_sha256, + }, + "typed_payload_files": 3, + "typed_payload_sha256": typed_sha256, + "public_api_sha256": public_sha256, + "public_names": ["Model", "Program", "Case"], + "pure_authoring": True, + "qualified_handles": True, + "py_typed": True, + "installed": True, + "installed_distribution": { + "name": "PoPS", + "version": "1.0.0", + "metadata_sha256": metadata_sha256, + }, + "installed_package": str(package), + "installed_typed_payload_sha256": typed_sha256, + "installed_public_api_sha256": public_sha256, + } + path = tmp_path / "public-api-evidence.json" + path.write_text(json.dumps(payload), encoding="utf-8") + release_evidence = { + "runtime": {"pops_file": str(package / "__init__.py")}, + "gates": { + "official_build": { + "evidence": {"wheel": {"sha256": wheel_sha256}}, + }, + }, + } + release = type("ReleaseContract", (), {"PACKAGE_VERSION": "1.0.0"}) + return path, release_evidence, release + + +def test_release_preflight_binds_installed_public_api_to_wheel_and_runtime(tmp_path): + evidence, release_evidence, release = _write_public_api_evidence(tmp_path) + + preflight._public_api_evidence(evidence, release_evidence, release) + + payload = json.loads(evidence.read_text(encoding="utf-8")) + payload["wheel_sha256"] = "e" * 64 + evidence.write_text(json.dumps(payload), encoding="utf-8") + with pytest.raises(preflight.PreflightError, match="another wheel"): + preflight._public_api_evidence(evidence, release_evidence, release) + + +def test_release_preflight_rejects_public_api_proven_on_another_install(tmp_path): + evidence, release_evidence, release = _write_public_api_evidence(tmp_path) + payload = json.loads(evidence.read_text(encoding="utf-8")) + payload["installed_package"] = str(tmp_path / "other" / "pops") + evidence.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(preflight.PreflightError, match="authenticated installed runtime"): + preflight._public_api_evidence(evidence, release_evidence, release) + + def test_tag_release_cannot_race_or_bypass_supported_matrix_wheel_and_final_gate(): release = (ROOT / ".github" / "workflows" / "release.yml").read_text() wheels = (ROOT / ".github" / "workflows" / "wheels.yml").read_text() diff --git a/tests/python/architecture/test_release_contract.py b/tests/python/architecture/test_release_contract.py index 96e565b6b..558415fbe 100644 --- a/tests/python/architecture/test_release_contract.py +++ b/tests/python/architecture/test_release_contract.py @@ -110,4 +110,7 @@ def test_release_mode_cannot_run_without_tag_install_and_authenticated_evidence( cwd=ROOT, text=True, capture_output=True, ) assert result.returncode != 0 - assert "requires --tag, --installed and --evidence" in result.stderr + assert ( + "requires --tag, --installed, --evidence and --public-api-evidence" + in result.stderr + ) From a9f7e231001cb47d61fbb852513952899f395ee9 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 01:18:34 +0200 Subject: [PATCH 102/109] release: bind codesign to published wheel bytes --- scripts/codesign_pops_extensions.py | 29 +++++++++++++++++++++++++++++ scripts/release_preflight.py | 6 ++++++ scripts/run_final_gate.py | 15 +++++++++++++-- 3 files changed, 48 insertions(+), 2 deletions(-) diff --git a/scripts/codesign_pops_extensions.py b/scripts/codesign_pops_extensions.py index 2a054e3d3..6681f47f1 100755 --- a/scripts/codesign_pops_extensions.py +++ b/scripts/codesign_pops_extensions.py @@ -72,6 +72,33 @@ def _checked_codesign(command: Sequence[str], *, action: str) -> subprocess.Comp return result +def _has_valid_adhoc_signature(codesign: str, extension: Path) -> bool: + """Return whether ``extension`` already carries the release signature policy. + + Release validation must not rewrite bytes which came from the retained wheel: those are the + bytes eventually published. Probe first and only repair an absent/invalid signature. The + release preflight separately refuses a repair which changes the retained native-member digest. + """ + + verification = subprocess.run( + (codesign, "--verify", "--strict", "--verbose=2", str(extension)), + text=True, + capture_output=True, + check=False, + ) + if verification.returncode != 0: + return False + inspection = subprocess.run( + (codesign, "--display", "--verbose=4", str(extension)), + text=True, + capture_output=True, + check=False, + ) + if inspection.returncode != 0: + return False + return "Signature=adhoc" in "%s\n%s" % (inspection.stdout, inspection.stderr) + + def codesign_imported_extensions(*, if_present: bool = False) -> tuple[Path, ...]: """Sign and verify every extension a clean ``import pops`` will load on Darwin.""" if sys.platform != "darwin": @@ -86,6 +113,8 @@ def codesign_imported_extensions(*, if_present: bool = False) -> tuple[Path, ... if not codesign: raise CodesignError("Darwin requires 'codesign', but it is not available on PATH") for extension in extensions: + if _has_valid_adhoc_signature(codesign, extension): + continue _checked_codesign( (codesign, "--force", "--sign", "-", str(extension)), action="ad-hoc signing %s" % extension) diff --git a/scripts/release_preflight.py b/scripts/release_preflight.py index 2e25a429c..f11a73784 100644 --- a/scripts/release_preflight.py +++ b/scripts/release_preflight.py @@ -398,6 +398,12 @@ def _codesign_evidence( "signature": "adhoc", }: raise PreflightError("codesign evidence does not authenticate the live native extension") + retained_native_sha256 = gates["installed_wheel"]["evidence"]["native_sha256"] + if extension["sha256"] != retained_native_sha256: + raise PreflightError( + "codesign changed the retained wheel native bytes; the published wheel " + "would differ from the validated runtime" + ) commands = row["commands"] logs = _command_evidence(directory, commands, gate="codesign") suffix = ["python", "scripts/codesign_pops_extensions.py", "--json"] diff --git a/scripts/run_final_gate.py b/scripts/run_final_gate.py index 0fbbbf2bc..4980627b5 100644 --- a/scripts/run_final_gate.py +++ b/scripts/run_final_gate.py @@ -200,7 +200,9 @@ def _json_evidence(stdout: str, *, gate: str) -> dict[str, Any]: return payload -def _signed_runtime_sha256(evidence: dict[str, Any]) -> str: +def _signed_runtime_sha256( + evidence: dict[str, Any], *, retained_native_sha256: str +) -> str: if set(evidence) != {"schema_version", "platform", "extensions"} \ or evidence["schema_version"] != 1 or evidence["platform"] != "darwin": raise FinalGateError("codesign evidence is not the Darwin release proof") @@ -215,6 +217,11 @@ def _signed_runtime_sha256(evidence: dict[str, Any]) -> str: if not isinstance(digest, str) or len(digest) != 64 \ or any(character not in "0123456789abcdef" for character in digest): raise FinalGateError("codesign extension sha256 is malformed") + if digest != retained_native_sha256: + raise FinalGateError( + "codesign changed the retained wheel native bytes; refusing to publish " + "an artifact different from the validated runtime" + ) return digest @@ -563,7 +570,11 @@ def main(argv: Sequence[str] | None = None) -> int: "selection": PYTHON_REQUIRED_SELECTION, } signed_runtime_sha256 = _signed_runtime_sha256( - recorder.rows["codesign"]["evidence"]) + recorder.rows["codesign"]["evidence"], + retained_native_sha256=( + recorder.rows["installed_wheel"]["evidence"]["native_sha256"] + ), + ) examples, reopened, restarted = _run_examples( recorder, runtime_sha256=signed_runtime_sha256) recorder.rows["examples"]["evidence"] = {"examples": examples} From b5a384fe59bc6f959ca41f4a77d8f47c6035d86e Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 01:18:43 +0200 Subject: [PATCH 103/109] test(release): refuse post-install signature drift --- .../test_codesign_build_scripts.py | 35 +++++++++++++++++-- .../architecture/test_final_release_gate.py | 18 ++++++++-- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/tests/python/architecture/test_codesign_build_scripts.py b/tests/python/architecture/test_codesign_build_scripts.py index 99807c62e..93066e027 100644 --- a/tests/python/architecture/test_codesign_build_scripts.py +++ b/tests/python/architecture/test_codesign_build_scripts.py @@ -56,7 +56,7 @@ def test_non_darwin_never_locates_or_invokes_codesign(monkeypatch): assert helper.codesign_imported_extensions() == () -def test_darwin_signs_then_verifies_and_authenticates_ad_hoc_signature(tmp_path, monkeypatch): +def test_darwin_preserves_an_existing_valid_ad_hoc_signature(tmp_path, monkeypatch): helper = _helper() extension = tmp_path / "_pops.so" extension.touch() @@ -74,6 +74,32 @@ def run(command, **kwargs): assert helper.codesign_imported_extensions() == (extension,) assert calls == [ + ("/usr/bin/codesign", "--verify", "--strict", "--verbose=2", str(extension)), + ("/usr/bin/codesign", "--display", "--verbose=4", str(extension)), + ] + + +def test_darwin_repairs_then_verifies_a_missing_signature(tmp_path, monkeypatch): + helper = _helper() + extension = tmp_path / "_pops.so" + extension.touch() + calls = [] + + def run(command, **kwargs): + calls.append(tuple(command)) + if len(calls) == 1: + return subprocess.CompletedProcess(command, 1, "", "unsigned") + evidence = "Signature=adhoc\n" if "--display" in command else "" + return subprocess.CompletedProcess(command, 0, "", evidence) + + monkeypatch.setattr(helper.sys, "platform", "darwin") + monkeypatch.setattr(helper, "locate_imported_pops_extensions", lambda: (extension,)) + monkeypatch.setattr(helper.shutil, "which", lambda command: "/usr/bin/codesign") + monkeypatch.setattr(helper.subprocess, "run", run) + + assert helper.codesign_imported_extensions() == (extension,) + assert calls == [ + ("/usr/bin/codesign", "--verify", "--strict", "--verbose=2", str(extension)), ("/usr/bin/codesign", "--force", "--sign", "-", str(extension)), ("/usr/bin/codesign", "--verify", "--strict", "--verbose=2", str(extension)), ("/usr/bin/codesign", "--display", "--verbose=4", str(extension)), @@ -101,7 +127,7 @@ def test_structured_evidence_binds_the_post_sign_extension_bytes(tmp_path, monke } -@pytest.mark.parametrize("failure_call", [0, 1]) +@pytest.mark.parametrize("failure_call", [1, 2, 3]) def test_darwin_codesign_or_verification_failure_is_explicit( tmp_path, monkeypatch, failure_call, ): @@ -113,9 +139,12 @@ def test_darwin_codesign_or_verification_failure_is_explicit( def run(command, **kwargs): call = len(calls) calls.append(tuple(command)) + if call == 0: + return subprocess.CompletedProcess(command, 1, "", "unsigned") if call == failure_call: return subprocess.CompletedProcess(command, 9, "", "signature failure") - return subprocess.CompletedProcess(command, 0, "", "") + evidence = "Signature=adhoc\n" if "--display" in command else "" + return subprocess.CompletedProcess(command, 0, "", evidence) monkeypatch.setattr(helper.sys, "platform", "darwin") monkeypatch.setattr(helper, "locate_imported_pops_extensions", lambda: (extension,)) diff --git a/tests/python/architecture/test_final_release_gate.py b/tests/python/architecture/test_final_release_gate.py index 1ed44ea76..be66df947 100644 --- a/tests/python/architecture/test_final_release_gate.py +++ b/tests/python/architecture/test_final_release_gate.py @@ -474,6 +474,9 @@ def test_release_preflight_binds_codesign_to_live_runtime(tmp_path): "native_sha256": "a" * 64, } gates = { + "installed_wheel": { + "evidence": {"native_sha256": runtime["native_sha256"]}, + }, "codesign": { "commands": [ { @@ -507,6 +510,11 @@ def test_release_preflight_binds_codesign_to_live_runtime(tmp_path): with pytest.raises(preflight.PreflightError, match="live native extension"): preflight._codesign_evidence(tmp_path, gates, runtime) + gates["codesign"]["evidence"]["extensions"][0]["sha256"] = runtime["native_sha256"] + gates["installed_wheel"]["evidence"]["native_sha256"] = "b" * 64 + with pytest.raises(preflight.PreflightError, match="published wheel"): + preflight._codesign_evidence(tmp_path, gates, runtime) + def test_installed_example_authenticates_native_bytes_before_execution( monkeypatch, tmp_path, capsys, @@ -552,7 +560,7 @@ def test_installed_example_authenticates_native_bytes_before_execution( assert "example_args=--output-dir|/proof/output" in output -def test_final_gate_rejects_incomplete_or_non_darwin_codesign_runtime(): +def test_final_gate_rejects_incomplete_non_darwin_or_rewritten_codesign_runtime(): evidence = { "schema_version": 1, "platform": "darwin", @@ -565,10 +573,14 @@ def test_final_gate_rejects_incomplete_or_non_darwin_codesign_runtime(): ], } - assert gate._signed_runtime_sha256(evidence) == "a" * 64 + assert gate._signed_runtime_sha256( + evidence, retained_native_sha256="a" * 64 + ) == "a" * 64 + with pytest.raises(gate.FinalGateError, match="different from the validated runtime"): + gate._signed_runtime_sha256(evidence, retained_native_sha256="b" * 64) evidence["platform"] = "linux" with pytest.raises(gate.FinalGateError, match="Darwin release proof"): - gate._signed_runtime_sha256(evidence) + gate._signed_runtime_sha256(evidence, retained_native_sha256="a" * 64) def test_release_preflight_requires_exact_runtime_bound_example_commands(tmp_path): From 687aa34339f4a0d938627fedeeb6707bd3049c95 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 01:18:50 +0200 Subject: [PATCH 104/109] docs(release): require signed wheel byte parity --- CHANGELOG.md | 3 +++ docs/VERSIONING.md | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b229b02b..89ed9decc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,9 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning ### Changed +- Release codesign now preserves an existing valid ad-hoc signature and refuses publication when + post-install signing changes the retained wheel's native bytes, so the published wheel and the + runtime exercised by conformance and final examples are byte-identical. - Strict AMR checkpoint payload v7 now persists the accepted shared-interface flux audit together with Program clocks, histories, tagging state, conservative ledger and synchronization report. Restart validates every fragment's topology epoch, level pair, exact clock window, resolved diff --git a/docs/VERSIONING.md b/docs/VERSIONING.md index d3673f179..3790bd07a 100644 --- a/docs/VERSIONING.md +++ b/docs/VERSIONING.md @@ -86,5 +86,9 @@ before the official build begins. `## [x.y.z] - YYYY-MM-DD` section. 3. Run `python scripts/generate_release_contract.py --check` and the release preflight; a missing build/codesign/example/conformance evidence record blocks tagging. + The Darwin gate first preserves an already-valid ad-hoc signature and requires the post-codesign + native digest to remain byte-identical to the retained wheel member. A repair confined to the + installed copy therefore blocks publication: the wheel users receive must itself contain the + exact signed runtime exercised by conformance and the final examples. 4. Merge, then `git tag vx.y.z` on master and `git push --tags`. The `release.yml` workflow turns the tag into a GitHub Release built from that CHANGELOG section. From cb46686eee893d9789ad750147d776097966042c Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 07:18:44 +0200 Subject: [PATCH 105/109] refactor(numerics): consume exact flux provider packs (ADC-682) --- CHANGELOG.md | 4 +- docs/ARCHITECTURE.md | 8 ++- include/pops/core/model/physical_model.hpp | 2 + include/pops/core/state/state.hpp | 30 ++++++-- include/pops/numerics/fv/flux_interfaces.hpp | 69 +++++++------------ .../spatial/primitives/state_access.hpp | 30 +++++--- include/pops/physics/bricks/hyperbolic.hpp | 44 ++++++------ .../pops/physics/composition/composite.hpp | 31 +++++---- include/pops/physics/fluids/euler.hpp | 10 +-- .../physics/advection_diffusion.hpp | 4 +- python/pops/codegen/module_emit_brick.py | 17 ++--- python/pops/codegen/module_emit_riemann.py | 16 +++-- python/pops/physics/_authoring_vars.py | 22 +++++- .../unit/numerics/test_flux_interfaces.cpp | 19 ++--- .../test_flux_interface_fences.py | 15 ++++ .../codegen/test_compiler_model_provider.py | 2 + tests/python/unit/codegen/test_dsl_brick.py | 10 +-- 17 files changed, 202 insertions(+), 131 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 10630f322..be1bcebb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,7 +45,9 @@ Format: [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); versioning installation; the builtin flux-register kernel follows the same reported contract. - Generated physical-flux bricks now make their qualified provider requirements executable native ABI evidence: the binder validates every row at compile time and reads only its declared storage - slots instead of scanning the model's complete auxiliary width. + slots instead of scanning the model's complete auxiliary width. Physical laws consume that exact + pack directly through compile-time provider reads; `PhysicalFluxView` no longer reconstructs a + process-wide `Aux` value. - AMR checkpoint capability reports now distinguish same-rank bit-identical replay from non-bit-identical rank-count rematerialization with Dense persisted histories. The explicit `RegridOnRestart()` policy now restores and authenticates the recorded accepted state before one diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index c6052ff58..e922bbc3d 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -801,9 +801,11 @@ model-qualified `FaceTrace` values plus `FaceContext` and returns a typed densit providers fail during selection; homonymous components from different owners never alias. Generated physical models carry those qualified rows as `flux_provider_requirements`. The native binder validates their count, qualification, availability, unique in-range storage slots and then -loads only those declared slots into the model-qualified device pack. Hand-written C++ test models -that do not declare this generated ABI retain the full-width fixture path; generated PoPS models -never use that route. +loads only those declared slots into the model-qualified device pack. The physical law reads that +pack directly through the bounded `flux_provider()` protocol: `PhysicalFluxView` never +reconstructs the global `Aux` source/implicit carrier. Hand-written C++ fixtures that do not declare +the generated ABI may populate a full-width test pack, but they execute through the same direct +physical-flux protocol. ## Limitations diff --git a/include/pops/core/model/physical_model.hpp b/include/pops/core/model/physical_model.hpp index d6b201c42..d6cd6733e 100644 --- a/include/pops/core/model/physical_model.hpp +++ b/include/pops/core/model/physical_model.hpp @@ -76,6 +76,8 @@ POPS_HD constexpr int aux_comps() { /// Requires: State, Aux == pops::Aux, n_vars, flux(u,a,dir), max_wave_speed(u,a,dir), /// source(u,a), elliptic_rhs(u). All these methods must be POPS_HD if called /// in kernels (not checked by the concept; responsibility of the author). +/// Finite-volume execution additionally instantiates the hyperbolic methods with the exact +/// BoundFluxProviders protocol; Aux remains the pointwise source/implicit carrier. /// Do not confuse with HyperbolicPhysicalModel which adds the variables and conversions. template concept PhysicalModel = diff --git a/include/pops/core/state/state.hpp b/include/pops/core/state/state.hpp index a49260cc3..0b7b6170d 100644 --- a/include/pops/core/state/state.hpp +++ b/include/pops/core/state/state.hpp @@ -103,6 +103,12 @@ POPS_HD StateVec operator*(Real s, StateVec a) { // on the DSL side (python/pops/dsl.py) if more than four named fields per model are wanted. inline constexpr int kAuxMaxExtra = 4; +// Width of the base provider channel and first model-named provider component. These constants +// precede Aux because both the legacy source-term carrier and the exact physical-flux provider pack +// implement the same compile-time read protocol. +inline constexpr int kAuxBaseComps = 3; +inline constexpr int kAuxNamedBase = kAuxBaseComps + 2; // = 5 (after B_z=3, T_e=4) + /// @brief POINTWISE auxiliary fields shared with the physics: single coupling channel. /// /// Role: carry to the point the outputs of the elliptic solver and the fields provided by the system, @@ -146,18 +152,32 @@ struct Aux { assert(k >= 0 && k < kAuxMaxExtra); return (k >= 0 && k < kAuxMaxExtra) ? extra[k] : std::numeric_limits::quiet_NaN(); } -}; -// Width of the aux channel of the base contract (phi, grad phi). A model reading additional -// fields declares a larger n_aux; cf. aux_comps()/load_aux(). -inline constexpr int kAuxBaseComps = 3; + /// Compile-time provider read used by pointwise physical laws. Source/implicit routes may still + /// carry Aux, while finite-volume fluxes pass the exact model-qualified pack; the law therefore + /// depends on this narrow read protocol rather than on either storage representation. + template + POPS_HD Real flux_provider() const { + static_assert(Component >= 0 && Component < kAuxNamedBase + kAuxMaxExtra, + "physical flux provider component is outside the declared native capability"); + if constexpr (Component == 0) + return phi; + else if constexpr (Component == 1) + return grad_x; + else if constexpr (Component == 2) + return grad_y; +#define POPS_AUX_PROVIDER_READ(name, index) else if constexpr (Component == index) return name; + POPS_AUX_FIELDS(POPS_AUX_PROVIDER_READ) +#undef POPS_AUX_PROVIDER_READ + else return extra[Component - kAuxNamedBase]; + } +}; // First component of the NAMED aux fields (ADC-70 phase 1): right AFTER the canonical fields // B_z (3) and T_e (4), so index 5. A model declaring K named fields sets n_aux = kAuxNamedBase + // K; extra[k] is component (kAuxNamedBase + k). Placed AFTER the canonical channel so that user // names never encroach on B_z / T_e (which keep their dedicated paths // set_magnetic_field / set_electron_temperature_from). Python MIRROR: AUX_NAMED_BASE (dsl.py). -inline constexpr int kAuxNamedBase = kAuxBaseComps + 2; // = 5 (after B_z=3, T_e=4) // Safeguard: the base of the named fields must be STRICTLY beyond the last canonical extra // field (the largest index of POPS_AUX_FIELDS + 1). If a canonical field is added beyond T_e, diff --git a/include/pops/numerics/fv/flux_interfaces.hpp b/include/pops/numerics/fv/flux_interfaces.hpp index 937f46dd5..52c812fd4 100644 --- a/include/pops/numerics/fv/flux_interfaces.hpp +++ b/include/pops/numerics/fv/flux_interfaces.hpp @@ -206,6 +206,13 @@ class BoundFluxProviders { POPS_HD BoundFluxProviders(const BoundFluxProviders&) = default; BoundFluxProviders& operator=(const BoundFluxProviders&) = delete; + template + POPS_HD Real flux_provider() const { + static_assert(Component >= 0 && Component < value_count, + "physical law requested a provider outside its exact qualified pack"); + return values_[Component]; + } + private: FluxProviderValues values_; @@ -352,9 +359,8 @@ POPS_HD IntegratedFaceFlux apply_face_measure(const FluxDensity& d } /// Narrow physical constitutive interface over a bound provider pack. Numerical-flux policies see -/// this value, never the complete runtime Model. The current native formulas still use Aux -/// internally; that storage representation is sealed behind BoundFluxProviders and cannot leak -/// into a numerical-flux signature. +/// this value, never the complete runtime Model. Physical laws consume BoundFluxProviders directly; +/// no global Aux value is reconstructed on the finite-volume path. template struct PhysicalFluxView { using State = typename Model::State; @@ -364,31 +370,8 @@ struct PhysicalFluxView { Model physical; - private: - POPS_HD static Aux physical_providers(const ProviderPack& providers) { - Aux result{}; - if constexpr (ProviderPack::value_count > 0) - result.phi = providers.values_[0]; - if constexpr (ProviderPack::value_count > 1) - result.grad_x = providers.values_[1]; - if constexpr (ProviderPack::value_count > 2) - result.grad_y = providers.values_[2]; -#define POPS_FLUX_PROVIDER_ASSIGN(name, index) \ - if constexpr (ProviderPack::value_count > index) \ - result.name = providers.values_[index]; - POPS_AUX_FIELDS(POPS_FLUX_PROVIDER_ASSIGN) -#undef POPS_FLUX_PROVIDER_ASSIGN - if constexpr (ProviderPack::value_count > kAuxNamedBase) { - for (int component = kAuxNamedBase; component < ProviderPack::value_count; ++component) - result.extra[component - kAuxNamedBase] = providers.values_[component]; - } - return result; - } - - public: POPS_HD FluxDensity evaluate(const Trace& trace, const FaceContext& face) const { - const Aux providers = physical_providers(trace.providers); - State result = physical.flux(trace.state, providers, face.axis); + State result = physical.flux(trace.state, trace.providers, face.axis); const Real sign = face.orientation_sign(); if (sign < Real(0)) { for (int component = 0; component < n_vars; ++component) @@ -398,18 +381,17 @@ struct PhysicalFluxView { } POPS_HD StabilityBound stability(const Trace& trace, const FaceContext& face) const { - const Aux providers = physical_providers(trace.providers); - return {physical.max_wave_speed(trace.state, providers, face.axis), + return {physical.max_wave_speed(trace.state, trace.providers, face.axis), StabilityUnit::kLengthPerTime, StabilityConvention::kNormalSpectralRadius}; } POPS_HD void signed_wave_speeds(const Trace& trace, const FaceContext& face, Real& lower, Real& upper) const - requires requires(const Model& model, const State& state, const Aux& providers, int axis, - Real& lo, Real& hi) { model.wave_speeds(state, providers, axis, lo, hi); } + requires requires(const Model& model, const State& state, const ProviderPack& providers, + int axis, Real& lo, + Real& hi) { model.wave_speeds(state, providers, axis, lo, hi); } { - const Aux providers = physical_providers(trace.providers); - physical.wave_speeds(trace.state, providers, face.axis, lower, upper); + physical.wave_speeds(trace.state, trace.providers, face.axis, lower, upper); if (face.orientation == FaceOrientation::kNegative) { const Real old_lower = lower; lower = -upper; @@ -444,12 +426,12 @@ struct PhysicalFluxView { POPS_HD State roe_dissipation(const Trace& left, const Trace& right, const FaceContext& face) const - requires requires(const Model& model, const State& l, const Aux& lp, const State& r, - const Aux& rp, int axis) { model.roe_dissipation(l, lp, r, rp, axis); } + requires requires(const Model& model, const State& l, const ProviderPack& lp, const State& r, + const ProviderPack& rp, + int axis) { model.roe_dissipation(l, lp, r, rp, axis); } { - const Aux left_values = physical_providers(left.providers); - const Aux right_values = physical_providers(right.providers); - return physical.roe_dissipation(left.state, left_values, right.state, right_values, face.axis); + return physical.roe_dissipation(left.state, left.providers, right.state, right.providers, + face.axis); } }; @@ -476,9 +458,9 @@ concept NumericalFlux = /// Constitutive capability gates used only during route resolution. NumericalFlux policies do not /// receive these Models; installation wraps a conforming value in the narrow PhysicalFluxView. template -concept HasHLLCStructure = requires(const Model& model, const typename Model::State& state, - const typename Model::State& other, const Aux& providers, - Real scalar, int axis, Real& lower, Real& upper) { +concept HasHLLCStructure = requires( + const Model& model, const typename Model::State& state, const typename Model::State& other, + const BoundFluxProviders& providers, Real scalar, int axis, Real& lower, Real& upper) { { model.pressure(state) } -> std::convertible_to; model.wave_speeds(state, providers, axis, lower, upper); { @@ -491,8 +473,9 @@ concept HasHLLCStructure = requires(const Model& model, const typename Model::St template concept HasRoeDissipation = - requires(const Model& model, const typename Model::State& left, const Aux& left_providers, - const typename Model::State& right, const Aux& right_providers, int axis) { + requires(const Model& model, const typename Model::State& left, + const BoundFluxProviders& left_providers, const typename Model::State& right, + const BoundFluxProviders& right_providers, int axis) { { model.roe_dissipation(left, left_providers, right, right_providers, axis) } -> std::same_as; diff --git a/include/pops/numerics/spatial/primitives/state_access.hpp b/include/pops/numerics/spatial/primitives/state_access.hpp index 7530ea03b..ec13204c4 100644 --- a/include/pops/numerics/spatial/primitives/state_access.hpp +++ b/include/pops/numerics/spatial/primitives/state_access.hpp @@ -53,9 +53,13 @@ struct SourceFreeModel { static constexpr int n_vars = M::n_vars; static constexpr int n_aux = aux_comps(); // transparent to the wrapped model's aux width M m; - POPS_HD State flux(const State& u, const Aux& a, int dir) const { return m.flux(u, a, dir); } - POPS_HD Real max_wave_speed(const State& u, const Aux& a, int dir) const { - return m.max_wave_speed(u, a, dir); + template + POPS_HD State flux(const State& u, const Providers& providers, int dir) const { + return m.flux(u, providers, dir); + } + template + POPS_HD Real max_wave_speed(const State& u, const Providers& providers, int dir) const { + return m.max_wave_speed(u, providers, dir); } POPS_HD State source(const State&, const Aux&) const { return State{}; } POPS_HD Real elliptic_rhs(const State& u) const { return m.elliptic_rhs(u); } @@ -69,12 +73,14 @@ struct SourceFreeModel { { return m.pressure(u); } - POPS_HD void wave_speeds(const State& u, const Aux& a, int dir, Real& smin, Real& smax) const - requires requires(const M& mm, const State& s, const Aux& aa, int d, Real& lo, Real& hi) { - mm.wave_speeds(s, aa, d, lo, hi); + template + POPS_HD void wave_speeds(const State& u, const Providers& providers, int dir, Real& smin, + Real& smax) const + requires requires(const M& mm, const State& s, const Providers& p, int d, Real& lo, Real& hi) { + mm.wave_speeds(s, p, d, lo, hi); } { - m.wave_speeds(u, a, dir, smin, smax); + m.wave_speeds(u, providers, dir, smin, smax); } // Roe / HLLC CAPABILITIES (HasRoeDissipation / HasHLLCStructure): forwarded ONLY if M exposes // them (requires clause), exactly like pressure / wave_speeds above and like composite.hpp. @@ -93,12 +99,14 @@ struct SourceFreeModel { { return m.hllc_star_state(u, p, s, sStar, dir); } - POPS_HD State roe_dissipation(const State& ul, const Aux& al, const State& ur, const Aux& ar, + template + POPS_HD State roe_dissipation(const State& ul, const LeftProviders& left_providers, + const State& ur, const RightProviders& right_providers, int dir) const - requires requires(const M& mm, const State a_, const Aux x_, const State b_, const Aux y_, - int d) { mm.roe_dissipation(a_, x_, b_, y_, d); } + requires requires(const M& mm, const State a_, const LeftProviders& x_, const State b_, + const RightProviders& y_, int d) { mm.roe_dissipation(a_, x_, b_, y_, d); } { - return m.roe_dissipation(ul, al, ur, ar, dir); + return m.roe_dissipation(ul, left_providers, ur, right_providers, dir); } // Forward the VariableSet introspection (HOST): lets positivity_comp resolve the Density role // through the explicit IMEX half-step. Conditional (requires), like pressure / wave_speeds. diff --git a/include/pops/physics/bricks/hyperbolic.hpp b/include/pops/physics/bricks/hyperbolic.hpp index 687cc6ebb..a380ecda2 100644 --- a/include/pops/physics/bricks/hyperbolic.hpp +++ b/include/pops/physics/bricks/hyperbolic.hpp @@ -28,22 +28,24 @@ struct ExBVelocity { static constexpr int n_vars = 1; using State = StateVec<1>; Real B0 = 1; - POPS_HD Real velocity(const Aux& a, int dir) const { - return (dir == 0) ? (-a.grad_y / B0) : (a.grad_x / B0); + POPS_HD Real velocity(const auto& providers, int dir) const { + const Real grad_x = providers.template flux_provider<1>(); + const Real grad_y = providers.template flux_provider<2>(); + return (dir == 0) ? (-grad_y / B0) : (grad_x / B0); } - POPS_HD StateVec<1> flux(const StateVec<1>& u, const Aux& a, int dir) const { + POPS_HD StateVec<1> flux(const StateVec<1>& u, const auto& providers, int dir) const { StateVec<1> f{}; - f[0] = u[0] * velocity(a, dir); + f[0] = u[0] * velocity(providers, dir); return f; } - POPS_HD Real max_wave_speed(const StateVec<1>&, const Aux& a, int dir) const { - const Real d = velocity(a, dir); + POPS_HD Real max_wave_speed(const StateVec<1>&, const auto& providers, int dir) const { + const Real d = velocity(providers, dir); return d < 0 ? -d : d; } /// Spectrum: one wave, the drift speed in direction dir. - POPS_HD StateVec<1> eigenvalues(const StateVec<1>&, const Aux& a, int dir) const { + POPS_HD StateVec<1> eigenvalues(const StateVec<1>&, const auto& providers, int dir) const { StateVec<1> e{}; - e[0] = velocity(a, dir); + e[0] = velocity(providers, dir); return e; } // Scalar: primitive variables = conservative (transported density). @@ -83,22 +85,24 @@ struct ExBVelocityPolar { using State = StateVec<1>; Real B0 = 1; /// PHYSICAL component of the drift velocity in direction index dir (0 = r, 1 = theta). - POPS_HD Real velocity(const Aux& a, int dir) const { - return (dir == 0) ? (-a.grad_y / B0) : (a.grad_x / B0); + POPS_HD Real velocity(const auto& providers, int dir) const { + const Real grad_x = providers.template flux_provider<1>(); + const Real grad_y = providers.template flux_provider<2>(); + return (dir == 0) ? (-grad_y / B0) : (grad_x / B0); } - POPS_HD StateVec<1> flux(const StateVec<1>& u, const Aux& a, int dir) const { + POPS_HD StateVec<1> flux(const StateVec<1>& u, const auto& providers, int dir) const { StateVec<1> f{}; - f[0] = u[0] * velocity(a, dir); + f[0] = u[0] * velocity(providers, dir); return f; } - POPS_HD Real max_wave_speed(const StateVec<1>&, const Aux& a, int dir) const { - const Real d = velocity(a, dir); + POPS_HD Real max_wave_speed(const StateVec<1>&, const auto& providers, int dir) const { + const Real d = velocity(providers, dir); return d < 0 ? -d : d; } /// Spectrum: one wave, the drift speed in direction dir. - POPS_HD StateVec<1> eigenvalues(const StateVec<1>&, const Aux& a, int dir) const { + POPS_HD StateVec<1> eigenvalues(const StateVec<1>&, const auto& providers, int dir) const { StateVec<1> e{}; - e[0] = velocity(a, dir); + e[0] = velocity(providers, dir); return e; } // Scalar: primitive variables = conservative (transported density). @@ -141,7 +145,7 @@ struct IsothermalFlux { POPS_HD Real velocity_rho(Real rho) const { return (vacuum_floor > Real(0) && rho < vacuum_floor) ? vacuum_floor : rho; } - POPS_HD StateVec<3> flux(const StateVec<3>& u, const Aux&, int dir) const { + POPS_HD StateVec<3> flux(const StateVec<3>& u, const auto&, int dir) const { const Real rho = u[0]; const Real vn = (dir == 0 ? u[1] : u[2]) / velocity_rho(rho); const Real p = cs2 * rho; @@ -169,14 +173,14 @@ struct IsothermalFlux { u[2] = p[0] * p[2]; return u; } - POPS_HD Real max_wave_speed(const StateVec<3>& u, const Aux&, int dir) const { + POPS_HD Real max_wave_speed(const StateVec<3>& u, const auto&, int dir) const { const Prim p = to_primitive(u); const Real vn = (dir == 0 ? p[1] : p[2]); const Real a = vn < 0 ? -vn : vn; return a + std::sqrt(cs2); } /// Full spectrum: (v_dir - c, v_dir, v_dir + c), c = sqrt(cs2). - POPS_HD StateVec<3> eigenvalues(const StateVec<3>& u, const Aux&, int dir) const { + POPS_HD StateVec<3> eigenvalues(const StateVec<3>& u, const auto&, int dir) const { const Prim p = to_primitive(u); const Real vn = (dir == 0 ? p[1] : p[2]); const Real c = std::sqrt(cs2); @@ -187,7 +191,7 @@ struct IsothermalFlux { return e; } /// Signed speeds (HLL/HLLC): v_dir -+ c_s. - POPS_HD void wave_speeds(const StateVec<3>& u, const Aux&, int dir, Real& smin, + POPS_HD void wave_speeds(const StateVec<3>& u, const auto&, int dir, Real& smin, Real& smax) const { const Prim p = to_primitive(u); const Real vn = (dir == 0 ? p[1] : p[2]); diff --git a/include/pops/physics/composition/composite.hpp b/include/pops/physics/composition/composite.hpp index aacb38bd6..6a8547eca 100644 --- a/include/pops/physics/composition/composite.hpp +++ b/include/pops/physics/composition/composite.hpp @@ -52,9 +52,13 @@ struct CompositeModel { Source src{}; Elliptic ell{}; - POPS_HD State flux(const State& u, const Aux& a, int dir) const { return hyp.flux(u, a, dir); } - POPS_HD Real max_wave_speed(const State& u, const Aux& a, int dir) const { - return hyp.max_wave_speed(u, a, dir); + template + POPS_HD State flux(const State& u, const Providers& providers, int dir) const { + return hyp.flux(u, providers, dir); + } + template + POPS_HD Real max_wave_speed(const State& u, const Providers& providers, int dir) const { + return hyp.max_wave_speed(u, providers, dir); } POPS_HD State source(const State& u, const Aux& a) const { return src.apply(u, a); } POPS_HD Real elliptic_rhs(const State& u) const { return ell.rhs(u); } @@ -68,12 +72,13 @@ struct CompositeModel { { return hyp.pressure(u); } - POPS_HD void wave_speeds(const State& u, const Aux& a, int dir, Real& smin, Real& smax) const - requires requires(const Hyperbolic h, const State s, const Aux aa, int d, Real& lo, Real& hi) { - h.wave_speeds(s, aa, d, lo, hi); - } + template + POPS_HD void wave_speeds(const State& u, const Providers& providers, int dir, Real& smin, + Real& smax) const + requires requires(const Hyperbolic h, const State s, const Providers& p, int d, Real& lo, + Real& hi) { h.wave_speeds(s, p, d, lo, hi); } { - hyp.wave_speeds(u, a, dir, smin, smax); + hyp.wave_speeds(u, providers, dir, smin, smax); } /// Riemann CAPABILITIES (audit wave 3): HLLC hooks (contact_speed + hllc_star_state) and Roe @@ -95,12 +100,14 @@ struct CompositeModel { { return hyp.hllc_star_state(u, p, s, sStar, dir); } - POPS_HD State roe_dissipation(const State& ul, const Aux& al, const State& ur, const Aux& ar, + template + POPS_HD State roe_dissipation(const State& ul, const LeftProviders& left_providers, + const State& ur, const RightProviders& right_providers, int dir) const - requires requires(const Hyperbolic h, const State a_, const Aux x_, const State b_, - const Aux y_, int d) { h.roe_dissipation(a_, x_, b_, y_, d); } + requires requires(const Hyperbolic h, const State a_, const LeftProviders& x_, const State b_, + const RightProviders& y_, int d) { h.roe_dissipation(a_, x_, b_, y_, d); } { - return hyp.roe_dissipation(ul, al, ur, ar, dir); + return hyp.roe_dissipation(ul, left_providers, ur, right_providers, dir); } /// GEOMETRIC source term of polar curvature, delegated to the hyperbolic brick when it diff --git a/include/pops/physics/fluids/euler.hpp b/include/pops/physics/fluids/euler.hpp index a1c1932d1..b4e2d92dc 100644 --- a/include/pops/physics/fluids/euler.hpp +++ b/include/pops/physics/fluids/euler.hpp @@ -81,7 +81,7 @@ struct Euler { * @param[out] smin leftmost wave speed v_dir - c * @param[out] smax rightmost wave speed v_dir + c */ - POPS_HD void wave_speeds(const State& u, const Aux&, int dir, Real& smin, Real& smax) const { + POPS_HD void wave_speeds(const State& u, const auto&, int dir, Real& smin, Real& smax) const { const Prim p = to_primitive(u); const Real vn = (dir == 0 ? p[1] : p[2]); const Real c = std::sqrt(gamma * p[3] / p[0]); @@ -90,7 +90,7 @@ struct Euler { } /// Compressible convective flux in direction dir. - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { const Real rho = u[0]; const Real vn = (dir == 0 ? u[1] : u[2]) / rho; // velocity normal to the face const Real p = pressure(u); @@ -139,7 +139,7 @@ struct Euler { /// eigenwave decomposition (F_R - F_L = A_roe (U_R - U_L) exactly), sqrt(rho) Roe average, gamma-1 /// from the ideal-gas EOS, and a typed Harten entropy policy on the acoustic waves. RoeFlux /// (HasRoeDissipation) then does F = 1/2 (F_L + F_R) - 1/2 d. - POPS_HD State roe_dissipation(const State& UL, const Aux&, const State& UR, const Aux&, + POPS_HD State roe_dissipation(const State& UL, const auto&, const State& UR, const auto&, int dir) const { const int in = (dir == 0) ? 1 : 2; // normal momentum const int it = (dir == 0) ? 2 : 1; // tangential @@ -186,7 +186,7 @@ struct Euler { /// Full spectrum in direction dir: (v_dir - c, v_dir, v_dir, v_dir + c). Vector counterpart /// of wave_speeds (which only gives the signed extremes); useful for spectrum schemes (Roe). - POPS_HD State eigenvalues(const State& u, const Aux&, int dir) const { + POPS_HD State eigenvalues(const State& u, const auto&, int dir) const { const Prim p = to_primitive(u); const Real vn = (dir == 0 ? p[1] : p[2]); const Real c = std::sqrt(gamma * p[3] / p[0]); @@ -199,7 +199,7 @@ struct Euler { } /// Maximum wave speed |v_dir| + c (Rusanov estimate), computed in primitive variables. - POPS_HD Real max_wave_speed(const State& u, const Aux&, int dir) const { + POPS_HD Real max_wave_speed(const State& u, const auto&, int dir) const { const Prim p = to_primitive(u); const Real vn = (dir == 0 ? p[1] : p[2]); const Real a = vn < 0 ? -vn : vn; // |v_dir| device-safe diff --git a/include/pops/validation/physics/advection_diffusion.hpp b/include/pops/validation/physics/advection_diffusion.hpp index f234d84c1..7508f69ec 100644 --- a/include/pops/validation/physics/advection_diffusion.hpp +++ b/include/pops/validation/physics/advection_diffusion.hpp @@ -42,11 +42,11 @@ struct AdvectionDiffusion { Real nu = 0.0; ///< diffusivity (0 = pure advection) /// Advection flux F = a u in direction dir. - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { return State{(dir == 0 ? ax : ay) * u[0]}; // F = a u } /// Maximum wave speed: magnitude of the advection velocity in direction dir. - POPS_HD Real max_wave_speed(const State&, const Aux&, int dir) const { + POPS_HD Real max_wave_speed(const State&, const auto&, int dir) const { const Real v = (dir == 0) ? ax : ay; return v < 0 ? -v : v; } diff --git a/python/pops/codegen/module_emit_brick.py b/python/pops/codegen/module_emit_brick.py index 7389c5e76..e21658674 100644 --- a/python/pops/codegen/module_emit_brick.py +++ b/python/pops/codegen/module_emit_brick.py @@ -71,11 +71,12 @@ def prim_locals(live: Any = None) -> list: return _prim_block(model, live, hoist_reciprocals) def aux_locals() -> list: - return model._aux_locals_lines() # canonical (a.) + named (a.extra_field(k)), ADC-70 + return model._flux_provider_locals_lines() - # Aux parameter named 'a' only if a formula reads an auxiliary field (canonical OR - # named ; otherwise anonymous, so as not to trigger an unused-parameter warning). - aux_param = "const Aux& a" if model._reads_aux() else "const Aux&" + # Physical laws consume the exact provider-read protocol. The parameter remains generic so + # direct pointwise callers may pass Aux while the FV route passes BoundFluxProviders + # without reconstructing the process-wide POD. + aux_param = "const auto& a" if model._reads_aux() else "const auto&" def eig_reduce(cpps: Any, ind: Any) -> list: # cpps : C++ already generated (possibly CSE) for the eigenvalues. Internal names suffixed @@ -271,11 +272,11 @@ def roles_init(roles: Any) -> Any: S += [" F[%d] = %s;" % (i, fcpps[nc + i]) for i in range(nc)] S += [" }", " return F;", " }", ""] - # in 'fd' jacobian mode WITHOUT eigenvalues, max_wave_speed calls flux(U, a, dir) : the - # Aux parameter must be named even if no formula reads an aux. + # In finite-difference Jacobian mode max_wave_speed calls flux(U, a, dir), so the provider + # parameter must be named even if no formula reads a provider directly. ws_jac: Any = model._ws_jacobian jac_fd = model._ws_jacobian is not None and model._ws_jacobian["eig"] == "fd" - mws_aux_param = "const Aux& a" if (jac_fd and not model._eig) else aux_param + mws_aux_param = "const auto& a" if (jac_fd and not model._eig) else aux_param S.append(" POPS_HD pops::Real max_wave_speed(const State& U, %s, int dir) const {" % mws_aux_param) if model._eig: @@ -389,7 +390,7 @@ def roles_init(roles: Any) -> Any: # flux ; extremes per sub-block via pops::real_eig_minmax. Non-convergence and non-real or # non-finite spectra invalidate the provider; the diagnostic Gershgorin enclosure is never # consumed as an HLL speed.) - ws_aux = aux_param if model._ws_jacobian["eig"] != "fd" else "const Aux& a" + ws_aux = aux_param if model._ws_jacobian["eig"] != "fd" else "const auto& a" S.append(" POPS_HD void wave_speeds(const State& U, %s, int dir, pops::Real& smin, " "pops::Real& smax) const {" % ws_aux) ws_drv = [] if model._ws_jacobian["eig"] == "fd" else _jac_entries(model) diff --git a/python/pops/codegen/module_emit_riemann.py b/python/pops/codegen/module_emit_riemann.py index 5c3a7d0f3..91a9d0ce7 100644 --- a/python/pops/codegen/module_emit_riemann.py +++ b/python/pops/codegen/module_emit_riemann.py @@ -19,6 +19,7 @@ from pops._dense_spectral import is_exact_block_triangular from pops.codegen.cpp_writer import _cpp_roe from pops.codegen.module_emit_helpers import ( + _AUX_CANONICAL, _codegen_exprs, _live_prims, _prim_block, @@ -144,8 +145,8 @@ def _emit_roe_roles(model: Any, nc: Any) -> list: passives = [c for c in range(nc) if c not in (iD, iX, iY, iE)] out.append(" // CAPABILITY ROE generee depuis les ROLES (enable_roe) : dissipation") out.append(" // |A_roe| dU du coeur generique (HasRoeDissipation), aucun layout fige.") - out.append(" POPS_HD State roe_dissipation(const State& UL, const pops::Aux&, " - "const State& UR, const pops::Aux&, int dir) const {") + out.append(" POPS_HD State roe_dissipation(const State& UL, const auto&, " + "const State& UR, const auto&, int dir) const {") out.append(" const int in_ = dir == 0 ? %d : %d;" % (iX, iY)) out.append(" const int it_ = dir == 0 ? %d : %d;" % (iY, iX)) out.append(" const pops::Real rL = UL[%d], rR = UR[%d];" % (iD, iD)) @@ -212,8 +213,8 @@ def _emit_roe_provided(model: Any, nc: Any) -> list: (guard at declaration and in check()).""" out = [] has_aux = bool(model.aux_names) # Aux parameters named aL/aR only if some aux exist - aL = "const pops::Aux& aL" if has_aux else "const pops::Aux&" - aR = "const pops::Aux& aR" if has_aux else "const pops::Aux&" + aL = "const auto& aL" if has_aux else "const auto&" + aR = "const auto& aR" if has_aux else "const auto&" out.append(" // CAPABILITY ROE FOURNIE (m.roe_dissipation) : dissipation d ecrite par") out.append(" // l'utilisateur via left()/right() des deux etats ; hook HasRoeDissipation.") out.append(" POPS_HD State roe_dissipation(const State& UL, %s, const State& UR, %s, " @@ -225,7 +226,8 @@ def _emit_roe_provided(model: Any, nc: Any) -> list: out += [" const pops::Real %s%s = %s;" % (side, p, _cpp_roe(e, side)) for p, e in model.prim_defs.items()] if has_aux: - out += [" const pops::Real %s%s = %s.%s;" % (side, n, av, n) + out += [" const pops::Real %s%s = %s.template flux_provider<%d>();" + % (side, n, av, _AUX_CANONICAL[n]) for n in model.aux_names] out.append(" State d{};") out.append(" if (dir == 0) {") @@ -260,8 +262,8 @@ def _emit_roe_jacobian(model: Any, nc: Any, cse: Any) -> list: else: out.append(" // Phi_delta(A), delta=%s ; spectre complexe/non converge refuse." % scalar_cpp(entropy_fix)) - out.append(" POPS_HD State roe_dissipation(const State& UL, const pops::Aux&, " - "const State& UR, const pops::Aux&, int dir) const {") + out.append(" POPS_HD State roe_dissipation(const State& UL, const auto&, " + "const State& UR, const auto&, int dir) const {") # conservatives at the ARITHMETIC-MEAN interface state Uavg = 1/2 (UL + UR) out += [" const pops::Real %s = pops::Real(0.5) * (UL[%d] + UR[%d]);" % (c, i, i) for i, c in enumerate(model.cons_names)] diff --git a/python/pops/physics/_authoring_vars.py b/python/pops/physics/_authoring_vars.py index 1c9fbeb72..b43852982 100644 --- a/python/pops/physics/_authoring_vars.py +++ b/python/pops/physics/_authoring_vars.py @@ -15,7 +15,7 @@ from pops._ir import Var, _wrap -from .aux import AUX_CANONICAL, AUX_NAMED_MAX, aux_total_n_aux +from .aux import AUX_CANONICAL, AUX_NAMED_BASE, AUX_NAMED_MAX, aux_total_n_aux if TYPE_CHECKING: from ._model_contract import _HyperbolicModel @@ -95,6 +95,26 @@ def _aux_locals_lines(self) -> Any: for k, n in enumerate(self.aux_extra_names)] return lines + def _flux_provider_locals_lines(self) -> Any: + """C++ locals read from the exact physical-flux provider protocol. + + Unlike ``_aux_locals_lines`` this emits no field access on the global ``pops::Aux`` POD. + Both ``pops::Aux`` (for non-FV pointwise callers) and ``BoundFluxProviders`` + implement ``flux_provider()``, so generated physical laws keep one formula and + the finite-volume route consumes only its resolved model-qualified pack. + """ + lines = [ + " const pops::Real %s = a.template flux_provider<%d>();" + % (name, AUX_CANONICAL[name]) + for name in self.aux_names + ] + lines += [ + " const pops::Real %s = a.template flux_provider<%d>();" + % (name, AUX_NAMED_BASE + index) + for index, name in enumerate(self.aux_extra_names) + ] + return lines + def _reads_aux(self) -> bool: """True if a formula reads an aux field (canonical or named): drives the naming of the Aux parameter ('a' vs anonymous) so as not to trigger an unused-parameter warning.""" diff --git a/tests/cpp/unit/numerics/test_flux_interfaces.cpp b/tests/cpp/unit/numerics/test_flux_interfaces.cpp index c8f5efa9f..d7bd9c57b 100644 --- a/tests/cpp/unit/numerics/test_flux_interfaces.cpp +++ b/tests/cpp/unit/numerics/test_flux_interfaces.cpp @@ -18,8 +18,8 @@ struct Advect { static constexpr int n_vars = 1; pops::Real speed = pops::Real(2); - POPS_HD State flux(const State& state, const Aux&, int) const { return State{state[0] * speed}; } - POPS_HD pops::Real max_wave_speed(const State&, const Aux&, int) const { + POPS_HD State flux(const State& state, const auto&, int) const { return State{state[0] * speed}; } + POPS_HD pops::Real max_wave_speed(const State&, const auto&, int) const { return speed < pops::Real(0) ? -speed : speed; } }; @@ -31,12 +31,12 @@ struct SelectiveInvalidAdvect { using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State& state, const Aux&, int) const { return State{state[0]}; } - POPS_HD pops::Real max_wave_speed(const State& state, const Aux&, int) const { + POPS_HD State flux(const State& state, const auto&, int) const { return State{state[0]}; } + POPS_HD pops::Real max_wave_speed(const State& state, const auto&, int) const { return state[0] == pops::Real(-1) ? std::numeric_limits::quiet_NaN() : pops::Real(2); } - POPS_HD void wave_speeds(const State& state, const Aux&, int, pops::Real& lower, + POPS_HD void wave_speeds(const State& state, const auto&, int, pops::Real& lower, pops::Real& upper) const { if (state[0] == pops::Real(-2)) { lower = upper = std::numeric_limits::quiet_NaN(); @@ -53,11 +53,12 @@ struct ProviderAdvect { static constexpr int n_vars = 1; static constexpr int n_aux = 3; - POPS_HD State flux(const State& state, const Aux& providers, int) const { - return State{state[0] * providers.grad_x}; + POPS_HD State flux(const State& state, const auto& providers, int) const { + return State{state[0] * providers.template flux_provider<1>()}; } - POPS_HD pops::Real max_wave_speed(const State&, const Aux& providers, int) const { - return providers.grad_x < pops::Real(0) ? -providers.grad_x : providers.grad_x; + POPS_HD pops::Real max_wave_speed(const State&, const auto& providers, int) const { + const pops::Real gradient = providers.template flux_provider<1>(); + return gradient < pops::Real(0) ? -gradient : gradient; } }; diff --git a/tests/python/architecture/test_flux_interface_fences.py b/tests/python/architecture/test_flux_interface_fences.py index 03efb02b7..0078f877f 100644 --- a/tests/python/architecture/test_flux_interface_fences.py +++ b/tests/python/architecture/test_flux_interface_fences.py @@ -48,6 +48,21 @@ def test_bound_native_flux_pack_is_exact_and_does_not_store_global_aux(): assert "FluxDensity checked_density() const" in header +def test_physical_flux_consumes_the_exact_pack_without_reconstructing_aux(): + header = _behavior(ROOT / "include/pops/numerics/fv/flux_interfaces.hpp") + physical = header.split("struct PhysicalFluxView", 2)[2].split("template ", 1)[0] + assert "physical_providers" not in physical + assert "Aux result" not in physical + assert "const Aux" not in physical + assert "trace.providers" in physical + assert "left.providers" in physical + assert "right.providers" in physical + + emitter = (ROOT / "python/pops/codegen/module_emit_brick.py").read_text(encoding="utf-8") + assert 'aux_param = "const auto& a"' in emitter + assert "_flux_provider_locals_lines" in emitter + + def test_generated_flux_pack_metadata_controls_native_storage_reads(): header = _behavior(ROOT / "include/pops/numerics/fv/flux_interfaces.hpp") assert "qualified_flux_provider_requirements_valid" in header diff --git a/tests/python/unit/codegen/test_compiler_model_provider.py b/tests/python/unit/codegen/test_compiler_model_provider.py index c915bb37c..135e2a521 100644 --- a/tests/python/unit/codegen/test_compiler_model_provider.py +++ b/tests/python/unit/codegen/test_compiler_model_provider.py @@ -134,6 +134,8 @@ def test_facade_and_formula_carrier_share_one_minimal_flux_provider_pack(): assert "true, 1" in source assert "static constexpr int n_flux_providers = 1;" in source assert "flux_provider_requirements" in source + assert "flux(const State& U, const auto& a, int dir)" in source + assert "a.template flux_provider<1>()" in source def test_field_dependent_flux_without_provider_fails_before_native_source(): diff --git a/tests/python/unit/codegen/test_dsl_brick.py b/tests/python/unit/codegen/test_dsl_brick.py index e2413aba8..fc9652085 100644 --- a/tests/python/unit/codegen/test_dsl_brick.py +++ b/tests/python/unit/codegen/test_dsl_brick.py @@ -58,8 +58,8 @@ def build_exb_brick(): """Transport scalaire par derive E x B (B0=1) : flux qui DEPEND des champs auxiliaires (grad phi). - Sert a verifier que la brique generee emet bien des locals aux (a.grad_x / a.grad_y) dans flux et - max_wave_speed, et reproduit la brique manuelle pops::ExBVelocity{B0=1}.""" + Sert a verifier que la brique generee lit le pack provider exact dans flux et max_wave_speed, + et reproduit la brique manuelle pops::ExBVelocity{B0=1}.""" e = HyperbolicModel("exb") (n,) = e.conservative_vars("n") gx = e.aux("grad_x") @@ -139,8 +139,10 @@ def main(): # (2) brique a flux dependant des AUXILIAIRES (ExB) : les locals aux doivent etre emis dans # flux ET max_wave_speed, et la brique doit egaler pops::ExBVelocity ecrite a la main. exb = build_exb_brick().emit_cpp_brick(name="ExBGen") - assert exb.count("const pops::Real grad_x = a.grad_x;") >= 2, "locals aux absents (flux/vitesse)" - assert "flux(const State& U, const Aux& a, int dir)" in exb, "parametre Aux non nomme dans le flux" + assert exb.count("const pops::Real grad_x = a.template flux_provider<1>();") >= 2, \ + "lectures provider absentes (flux/vitesse)" + assert "flux(const State& U, const auto& a, int dir)" in exb, \ + "parametre provider exact non nomme dans le flux" prog2 = EXB_HARNESS % exb with tempfile.TemporaryDirectory() as tmp: cpp = os.path.join(tmp, "exb.cpp") From f47431c377d562c8b75b66ff065b1080c4e10afd Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 07:18:58 +0200 Subject: [PATCH 106/109] test(numerics): migrate flux fixtures to provider protocol --- .../amr/test_amr_composite_poisson.cpp | 4 ++-- .../integration/amr/test_amr_diagnostics.cpp | 7 +++--- .../integration/amr/test_amr_history_ring.cpp | 4 ++-- .../integration/amr/test_amr_layout_guard.cpp | 4 ++-- .../amr/test_amr_multiblock_imex.cpp | 4 ++-- .../amr/test_amr_multiblock_substeps.cpp | 4 ++-- .../amr/test_amr_program_diffusion.cpp | 4 ++-- .../amr/test_amr_program_positivity_floor.cpp | 4 ++-- .../amr/test_amr_system_bz_multibox.cpp | 8 +++---- .../amr/test_amr_system_bz_pop.cpp | 8 +++---- .../mpi/test_mpi_system_layout_transfer.cpp | 4 ++-- .../test_flux_failure_loader_transaction.cpp | 4 ++-- .../native_loader/test_native_aux_named.cpp | 4 ++-- .../runtime/test_aux_system_bz.cpp | 8 +++---- .../runtime/test_system_abstraction.cpp | 8 +++---- .../runtime/test_system_coupler.cpp | 8 +++---- .../runtime/test_system_hardening.cpp | 4 ++-- .../runtime/test_system_two_explicit.cpp | 8 +++---- .../test_wave_speed_cache_engagement.cpp | 6 ++--- .../elliptic/test_elliptic_composite_rhs.cpp | 4 ++-- .../unit/elliptic/test_newton_robustness.cpp | 16 ++++++------- tests/cpp/unit/numerics/test_cfl_dt.cpp | 10 ++++---- tests/cpp/unit/numerics/test_diffusion.cpp | 4 ++-- tests/cpp/unit/numerics/test_imex_partial.cpp | 4 ++-- .../cpp/unit/numerics/test_imex_transport.cpp | 4 ++-- .../unit/numerics/test_positivity_floor.cpp | 4 ++-- .../numerics/test_riemann_capabilities.cpp | 24 +++++++++---------- .../unit/numerics/test_weno_convergence.cpp | 4 ++-- .../unit/physics/test_adaptive_multirate.cpp | 4 ++-- .../cpp/unit/physics/test_aux_coupler_bz.cpp | 4 ++-- tests/cpp/unit/physics/test_aux_extra.cpp | 8 +++---- .../unit/physics/test_multirate_stride.cpp | 4 ++-- tests/cpp/unit/physics/test_polar_mms_vr.cpp | 10 ++++---- .../unit/physics/test_two_species_minimal.cpp | 8 +++---- .../physics/test_user_time_integrator.cpp | 4 ++-- .../unit/runtime/test_assembler_driver.cpp | 4 ++-- .../cpp/unit/runtime/test_coupled_source.cpp | 4 ++-- .../unit/runtime/test_disc_domain_mask.cpp | 6 ++--- tests/cpp/unit/runtime/test_eb_transport.cpp | 6 ++--- .../test_embedded_boundary_generic.cpp | 4 ++-- tests/gpu/romeo/gpu_aux_validate.cpp | 4 ++-- 41 files changed, 126 insertions(+), 123 deletions(-) diff --git a/tests/cpp/integration/amr/test_amr_composite_poisson.cpp b/tests/cpp/integration/amr/test_amr_composite_poisson.cpp index 427ce115d..2656120dc 100644 --- a/tests/cpp/integration/amr/test_amr_composite_poisson.cpp +++ b/tests/cpp/integration/amr/test_amr_composite_poisson.cpp @@ -46,8 +46,8 @@ struct ScalarCharge { using State = StateVec<1>; using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/integration/amr/test_amr_diagnostics.cpp b/tests/cpp/integration/amr/test_amr_diagnostics.cpp index 285d7f3fd..d9f1f88ab 100644 --- a/tests/cpp/integration/amr/test_amr_diagnostics.cpp +++ b/tests/cpp/integration/amr/test_amr_diagnostics.cpp @@ -54,12 +54,13 @@ struct DiagnosticWaveModel { static constexpr int n_vars = 1; Real B0 = Real(2); - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } POPS_HD State source(const State&, const Aux&) const { return State{}; } POPS_HD Real elliptic_rhs(const State&) const { return Real(0); } - POPS_HD Real max_wave_speed(const State& state, const Aux& aux, int direction) const { + POPS_HD Real max_wave_speed(const State& state, const auto& providers, int direction) const { const Real state_magnitude = state[0] < Real(0) ? -state[0] : state[0]; - const Real gradient = direction == 0 ? aux.grad_x : aux.grad_y; + const Real gradient = direction == 0 ? providers.template flux_provider<1>() + : providers.template flux_provider<2>(); const Real gradient_magnitude = gradient < Real(0) ? -gradient : gradient; return Real(direction + 1) * state_magnitude + gradient_magnitude; } diff --git a/tests/cpp/integration/amr/test_amr_history_ring.cpp b/tests/cpp/integration/amr/test_amr_history_ring.cpp index 08ca780e7..fa5fb8d12 100644 --- a/tests/cpp/integration/amr/test_amr_history_ring.cpp +++ b/tests/cpp/integration/amr/test_amr_history_ring.cpp @@ -65,8 +65,8 @@ struct QuadraticGrowthModel { using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State& u, const Aux&) const { return State{u[0] * u[0]}; } POPS_HD Real elliptic_rhs(const State&) const { return Real(0); } POPS_HD Prim to_primitive(const State& state) const { return state; } diff --git a/tests/cpp/integration/amr/test_amr_layout_guard.cpp b/tests/cpp/integration/amr/test_amr_layout_guard.cpp index 5095ff6be..d568e893f 100644 --- a/tests/cpp/integration/amr/test_amr_layout_guard.cpp +++ b/tests/cpp/integration/amr/test_amr_layout_guard.cpp @@ -43,10 +43,10 @@ struct AdvectX { using Aux = pops::Aux; static constexpr int n_vars = 1; Real a = Real(1); - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { return State{dir == 0 ? a * u[0] : Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return a < 0 ? -a : a; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return a < 0 ? -a : a; } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/integration/amr/test_amr_multiblock_imex.cpp b/tests/cpp/integration/amr/test_amr_multiblock_imex.cpp index 3c4ffcb2d..4387d4c1a 100644 --- a/tests/cpp/integration/amr/test_amr_multiblock_imex.cpp +++ b/tests/cpp/integration/amr/test_amr_multiblock_imex.cpp @@ -121,8 +121,8 @@ struct NonlinearDensityDecay { Real rate = Real(0); - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State& u, const Aux&) const { return State{-rate * u[0] * u[0]}; } POPS_HD Real elliptic_rhs(const State&) const { return Real(0); } POPS_HD Prim to_primitive(const State& state) const { return state; } diff --git a/tests/cpp/integration/amr/test_amr_multiblock_substeps.cpp b/tests/cpp/integration/amr/test_amr_multiblock_substeps.cpp index 58b932c72..a6b63b66f 100644 --- a/tests/cpp/integration/amr/test_amr_multiblock_substeps.cpp +++ b/tests/cpp/integration/amr/test_amr_multiblock_substeps.cpp @@ -177,8 +177,8 @@ struct TemporalContractModel { static constexpr int n_vars = 1; int mode = 0; - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State& u, const Aux&, int) const { + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State& u, const auto&, int) const { return mode == 1 ? (u[0] < Real(0) ? -u[0] : u[0]) : Real(0); } POPS_HD State source(const State& u, const Aux&) const { return State{u[0]}; } diff --git a/tests/cpp/integration/amr/test_amr_program_diffusion.cpp b/tests/cpp/integration/amr/test_amr_program_diffusion.cpp index ad4359374..225f7e913 100644 --- a/tests/cpp/integration/amr/test_amr_program_diffusion.cpp +++ b/tests/cpp/integration/amr/test_amr_program_diffusion.cpp @@ -38,8 +38,8 @@ struct DiffusiveScalar { Real nu = Real(0); - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } POPS_HD Real elliptic_rhs(const State&) const { return Real(0); } POPS_HD Real diffusivity() const { return nu; } diff --git a/tests/cpp/integration/amr/test_amr_program_positivity_floor.cpp b/tests/cpp/integration/amr/test_amr_program_positivity_floor.cpp index 9e65a3a88..2b6865549 100644 --- a/tests/cpp/integration/amr/test_amr_program_positivity_floor.cpp +++ b/tests/cpp/integration/amr/test_amr_program_positivity_floor.cpp @@ -39,10 +39,10 @@ struct DensityAdvection { using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State& state, const Aux&, int direction) const { + POPS_HD State flux(const State& state, const auto&, int direction) const { return direction == 0 ? state : State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int direction) const { + POPS_HD Real max_wave_speed(const State&, const auto&, int direction) const { return direction == 0 ? Real(1) : Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } diff --git a/tests/cpp/integration/amr/test_amr_system_bz_multibox.cpp b/tests/cpp/integration/amr/test_amr_system_bz_multibox.cpp index 0d09ce025..3edf851e0 100644 --- a/tests/cpp/integration/amr/test_amr_system_bz_multibox.cpp +++ b/tests/cpp/integration/amr/test_amr_system_bz_multibox.cpp @@ -54,8 +54,8 @@ struct BzGrowMB { using Aux = pops::Aux; static constexpr int n_vars = 1; static constexpr int n_aux = 4; // phi, grad_x, grad_y, B_z - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State& u, const Aux& a) const { return State{a.B_z * u[0]}; } POPS_HD Real elliptic_rhs(const State&) const { return Real(0); } }; @@ -65,8 +65,8 @@ struct InertMB { using State = StateVec<1>; using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{}; } POPS_HD Real elliptic_rhs(const State&) const { return Real(0); } }; diff --git a/tests/cpp/integration/amr/test_amr_system_bz_pop.cpp b/tests/cpp/integration/amr/test_amr_system_bz_pop.cpp index cc3aba84a..91c3b0e8a 100644 --- a/tests/cpp/integration/amr/test_amr_system_bz_pop.cpp +++ b/tests/cpp/integration/amr/test_amr_system_bz_pop.cpp @@ -50,8 +50,8 @@ struct BzGrowPop { using Aux = pops::Aux; static constexpr int n_vars = 1; static constexpr int n_aux = 4; // phi, grad_x, grad_y, B_z - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State& u, const Aux& a) const { return State{a.B_z * u[0]}; } POPS_HD Real elliptic_rhs(const State&) const { return Real(0); } }; @@ -62,10 +62,10 @@ struct AdvectXPop { using Aux = pops::Aux; static constexpr int n_vars = 1; Real v = Real(1); - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { return State{dir == 0 ? v * u[0] : Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return std::fabs(v); } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return std::fabs(v); } POPS_HD State source(const State&, const Aux&) const { return State{}; } POPS_HD Real elliptic_rhs(const State&) const { return Real(0); } }; diff --git a/tests/cpp/integration/mpi/test_mpi_system_layout_transfer.cpp b/tests/cpp/integration/mpi/test_mpi_system_layout_transfer.cpp index ca0243838..28bb85225 100644 --- a/tests/cpp/integration/mpi/test_mpi_system_layout_transfer.cpp +++ b/tests/cpp/integration/mpi/test_mpi_system_layout_transfer.cpp @@ -141,8 +141,8 @@ struct PassiveScalar { using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD pops::Real max_wave_speed(const State&, const Aux&, int) const { return pops::Real(1); } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD pops::Real max_wave_speed(const State&, const auto&, int) const { return pops::Real(1); } POPS_HD State source(const State&, const Aux&) const { return State{}; } POPS_HD pops::Real elliptic_rhs(const State&) const { return pops::Real(0); } POPS_HD Prim to_primitive(const State& state) const { return state; } diff --git a/tests/cpp/integration/native_loader/test_flux_failure_loader_transaction.cpp b/tests/cpp/integration/native_loader/test_flux_failure_loader_transaction.cpp index 9029d8818..dd1dd5ad3 100644 --- a/tests/cpp/integration/native_loader/test_flux_failure_loader_transaction.cpp +++ b/tests/cpp/integration/native_loader/test_flux_failure_loader_transaction.cpp @@ -64,8 +64,8 @@ std::string package_source() { using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD pops::Real max_wave_speed(const State&, const Aux&, int) const { return pops::Real(1); } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD pops::Real max_wave_speed(const State&, const auto&, int) const { return pops::Real(1); } POPS_HD State source(const State& state, const Aux&) const { return State{-state[0]}; } POPS_HD pops::Real elliptic_rhs(const State&) const { return pops::Real(0); } POPS_HD Prim to_primitive(const State& state) const { return state; } diff --git a/tests/cpp/integration/native_loader/test_native_aux_named.cpp b/tests/cpp/integration/native_loader/test_native_aux_named.cpp index 01c8a2121..118ad382f 100644 --- a/tests/cpp/integration/native_loader/test_native_aux_named.cpp +++ b/tests/cpp/integration/native_loader/test_native_aux_named.cpp @@ -35,8 +35,8 @@ std::string package_source() { using Aux = pops::Aux; static constexpr int n_vars = 1; static constexpr int n_aux = pops::kAuxNamedBase + 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD pops::Real max_wave_speed(const State&, const Aux&, int) const { return pops::Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD pops::Real max_wave_speed(const State&, const auto&, int) const { return pops::Real(0); } POPS_HD State source(const State& u, const Aux& aux) const { return State{aux.extra_field(0) * u[0]}; } diff --git a/tests/cpp/integration/runtime/test_aux_system_bz.cpp b/tests/cpp/integration/runtime/test_aux_system_bz.cpp index 09693d775..8825b8d83 100644 --- a/tests/cpp/integration/runtime/test_aux_system_bz.cpp +++ b/tests/cpp/integration/runtime/test_aux_system_bz.cpp @@ -28,8 +28,8 @@ struct BzGrow { using Aux = pops::Aux; static constexpr int n_vars = 1; static constexpr int n_aux = 4; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State& u, const Aux& a) const { State s{}; s[0] = a.B_z * u[0]; @@ -43,8 +43,8 @@ struct Scalar { using State = StateVec<1>; using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{}; } POPS_HD Real elliptic_rhs(const State&) const { return Real(0); } }; diff --git a/tests/cpp/integration/runtime/test_system_abstraction.cpp b/tests/cpp/integration/runtime/test_system_abstraction.cpp index 1ac45204c..df09ee6bc 100644 --- a/tests/cpp/integration/runtime/test_system_abstraction.cpp +++ b/tests/cpp/integration/runtime/test_system_abstraction.cpp @@ -22,8 +22,8 @@ struct ElectronToy { using State = StateVec<1>; using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } POPS_HD Real elliptic_rhs(const State& u) const { return -u[0]; } }; @@ -32,8 +32,8 @@ struct IonToy { using State = StateVec<1>; using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/integration/runtime/test_system_coupler.cpp b/tests/cpp/integration/runtime/test_system_coupler.cpp index e3efd5dc7..ba6a951d6 100644 --- a/tests/cpp/integration/runtime/test_system_coupler.cpp +++ b/tests/cpp/integration/runtime/test_system_coupler.cpp @@ -25,8 +25,8 @@ struct ElectronSource { Real rate = Real(2); - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{rate}; } POPS_HD Real elliptic_rhs(const State& u) const { return -u[0]; } }; @@ -38,8 +38,8 @@ struct IonSource { Real rate = Real(3); - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{rate}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/integration/runtime/test_system_hardening.cpp b/tests/cpp/integration/runtime/test_system_hardening.cpp index d25ed1258..8bed1fa6a 100644 --- a/tests/cpp/integration/runtime/test_system_hardening.cpp +++ b/tests/cpp/integration/runtime/test_system_hardening.cpp @@ -25,8 +25,8 @@ struct Scalar { using State = StateVec<1>; using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/integration/runtime/test_system_two_explicit.cpp b/tests/cpp/integration/runtime/test_system_two_explicit.cpp index 6b945b908..8282fa7a8 100644 --- a/tests/cpp/integration/runtime/test_system_two_explicit.cpp +++ b/tests/cpp/integration/runtime/test_system_two_explicit.cpp @@ -31,8 +31,8 @@ struct Production { using Aux = pops::Aux; static constexpr int n_vars = 1; Real rate = Real(1); - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{rate}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; @@ -44,10 +44,10 @@ struct AdvectX { using Aux = pops::Aux; static constexpr int n_vars = 1; Real a = Real(1); - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { return State{dir == 0 ? a * u[0] : Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return a < 0 ? -a : a; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return a < 0 ? -a : a; } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/integration/runtime/test_wave_speed_cache_engagement.cpp b/tests/cpp/integration/runtime/test_wave_speed_cache_engagement.cpp index 392595642..44ac6878b 100644 --- a/tests/cpp/integration/runtime/test_wave_speed_cache_engagement.cpp +++ b/tests/cpp/integration/runtime/test_wave_speed_cache_engagement.cpp @@ -46,7 +46,7 @@ struct CountingIsothermal { int busy = 0; Counter calls; // handle capture par valeur dans le kernel (donnees partagees) - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { const Real rho = u[0]; const Real vx = u[1] / rho, vy = u[2] / rho; const Real p = c0 * c0 * rho; @@ -62,12 +62,12 @@ struct CountingIsothermal { } return F; } - POPS_HD Real max_wave_speed(const State& u, const Aux&, int dir) const { + POPS_HD Real max_wave_speed(const State& u, const auto&, int dir) const { const Real v = (dir == 0 ? u[1] : u[2]) / u[0]; const Real av = v < 0 ? -v : v; return av + c0; } - POPS_HD void wave_speeds(const State& u, const Aux&, int dir, Real& lo, Real& hi) const { + POPS_HD void wave_speeds(const State& u, const auto&, int dir, Real& lo, Real& hi) const { Kokkos::atomic_add(&calls(), 1LL); const Real v = (dir == 0 ? u[1] : u[2]) / u[0]; Real acc = Real(0); diff --git a/tests/cpp/unit/elliptic/test_elliptic_composite_rhs.cpp b/tests/cpp/unit/elliptic/test_elliptic_composite_rhs.cpp index ee3cec6db..9524b2bc3 100644 --- a/tests/cpp/unit/elliptic/test_elliptic_composite_rhs.cpp +++ b/tests/cpp/unit/elliptic/test_elliptic_composite_rhs.cpp @@ -37,8 +37,8 @@ struct ScalarElliptic { using Aux = pops::Aux; static constexpr int n_vars = 1; Elliptic ell{}; - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } POPS_HD Real elliptic_rhs(const State& u) const { return ell.rhs(u); } }; diff --git a/tests/cpp/unit/elliptic/test_newton_robustness.cpp b/tests/cpp/unit/elliptic/test_newton_robustness.cpp index 4376a8753..f226121bb 100644 --- a/tests/cpp/unit/elliptic/test_newton_robustness.cpp +++ b/tests/cpp/unit/elliptic/test_newton_robustness.cpp @@ -33,8 +33,8 @@ struct StiffModel { using Aux = pops::Aux; static constexpr int n_vars = 3; Real k = 200.0; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return 0; } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return 0; } POPS_HD State source(const State& u, const Aux&) const { State s{}; s[0] = -k * (u[0] - u[1] * u[2]); @@ -67,8 +67,8 @@ struct NanModel { using State = pops::StateVec<3>; using Aux = pops::Aux; static constexpr int n_vars = 3; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return 0; } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return 0; } POPS_HD State source(const State& u, const Aux&) const { State s{}; s[0] = -u[0]; @@ -85,8 +85,8 @@ struct SingularModel { using State = pops::StateVec<3>; using Aux = pops::Aux; static constexpr int n_vars = 3; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return 0; } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return 0; } POPS_HD State source(const State& u, const Aux&) const { State s{}; s[0] = Real(8) * u[0] + Real(1); @@ -132,8 +132,8 @@ struct FallibleSourceModel { pops::ImplicitEvaluationStatus evaluation = pops::ImplicitEvaluationStatus::kOk; std::uint32_t reason = 0; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return 0; } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return 0; } POPS_HD State source(const State&, const Aux&) const { return State{Real(1e6), Real(1e6), Real(1e6)}; } diff --git a/tests/cpp/unit/numerics/test_cfl_dt.cpp b/tests/cpp/unit/numerics/test_cfl_dt.cpp index eba10e80d..554a04a24 100644 --- a/tests/cpp/unit/numerics/test_cfl_dt.cpp +++ b/tests/cpp/unit/numerics/test_cfl_dt.cpp @@ -30,10 +30,10 @@ struct AdvectX { using Aux = pops::Aux; static constexpr int n_vars = 1; Real a = Real(1); - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { return State{dir == 0 ? a * u[0] : Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return a < 0 ? -a : a; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return a < 0 ? -a : a; } POPS_HD State source(const State&, const Aux&) const { return State{}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; @@ -44,8 +44,8 @@ struct NanSpeed { using State = StateVec<1>; using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return std::numeric_limits::quiet_NaN(); } POPS_HD State source(const State&, const Aux&) const { return State{}; } @@ -63,7 +63,7 @@ struct BoundProbe { Real frequency = Real(0); Real direct_dt = std::numeric_limits::infinity(); - POPS_HD Real max_wave_speed(const State&, const Aux&, int direction) const { + POPS_HD Real max_wave_speed(const State&, const auto&, int direction) const { return direction == 0 ? wave_x : wave_y; } POPS_HD Real stability_speed(const State&, const Aux&, int direction) const { diff --git a/tests/cpp/unit/numerics/test_diffusion.cpp b/tests/cpp/unit/numerics/test_diffusion.cpp index fbc8cc80d..1b70f6331 100644 --- a/tests/cpp/unit/numerics/test_diffusion.cpp +++ b/tests/cpp/unit/numerics/test_diffusion.cpp @@ -33,8 +33,8 @@ struct Heat { using Aux = pops::Aux; static constexpr int n_vars = 1; Real nu = 0.0; - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } POPS_HD Real elliptic_rhs(const State&) const { return Real(0); } POPS_HD Real diffusivity() const { return nu; } diff --git a/tests/cpp/unit/numerics/test_imex_partial.cpp b/tests/cpp/unit/numerics/test_imex_partial.cpp index 1306c65bf..2d532697d 100644 --- a/tests/cpp/unit/numerics/test_imex_partial.cpp +++ b/tests/cpp/unit/numerics/test_imex_partial.cpp @@ -29,8 +29,8 @@ struct TwoVarRelax { using State = StateVec<2>; using Aux = pops::Aux; static constexpr int n_vars = 2; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State& u, const Aux&) const { return State{-Real(100) * (u[0] - Real(1)), -Real(1) * (u[1] - Real(2))}; } diff --git a/tests/cpp/unit/numerics/test_imex_transport.cpp b/tests/cpp/unit/numerics/test_imex_transport.cpp index 3fe2058e4..a8f15273d 100644 --- a/tests/cpp/unit/numerics/test_imex_transport.cpp +++ b/tests/cpp/unit/numerics/test_imex_transport.cpp @@ -28,10 +28,10 @@ struct AdvectX { using Aux = pops::Aux; static constexpr int n_vars = 1; Real a = Real(1); - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { return State{dir == 0 ? a * u[0] : Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return a < 0 ? -a : a; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return a < 0 ? -a : a; } POPS_HD State source(const State&, const Aux&) const { return State{}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/unit/numerics/test_positivity_floor.cpp b/tests/cpp/unit/numerics/test_positivity_floor.cpp index 0caed89c6..c9f9c7e89 100644 --- a/tests/cpp/unit/numerics/test_positivity_floor.cpp +++ b/tests/cpp/unit/numerics/test_positivity_floor.cpp @@ -47,8 +47,8 @@ struct EulerNoSrc { static constexpr int n_vars = Euler::n_vars; Euler e{}; Real gamma = Real(1.4); - POPS_HD State flux(const State& u, const Aux& a, int dir) const { return e.flux(u, a, dir); } - POPS_HD Real max_wave_speed(const State& u, const Aux& a, int dir) const { + POPS_HD State flux(const State& u, const auto& a, int dir) const { return e.flux(u, a, dir); } + POPS_HD Real max_wave_speed(const State& u, const auto& a, int dir) const { return e.max_wave_speed(u, a, dir); } POPS_HD State source(const State&, const Aux&) const { return State{}; } diff --git a/tests/cpp/unit/numerics/test_riemann_capabilities.cpp b/tests/cpp/unit/numerics/test_riemann_capabilities.cpp index 6068d5f55..0290f1178 100644 --- a/tests/cpp/unit/numerics/test_riemann_capabilities.cpp +++ b/tests/cpp/unit/numerics/test_riemann_capabilities.cpp @@ -48,7 +48,7 @@ struct HookedEuler : pops::Euler { Us[3] = fac * (U[3] / r + (sStar - un) * (sStar + p / (r * (s - un)))); return Us; } - POPS_HD State roe_dissipation(const State& UL, const Aux&, const State& UR, const Aux&, + POPS_HD State roe_dissipation(const State& UL, const auto&, const State& UR, const auto&, int dir) const { const int in = (dir == 0) ? 1 : 2; const int it = (dir == 0) ? 2 : 1; @@ -101,13 +101,13 @@ struct PermutedEuler { return State{value[3], value[2], value[0], value[1]}; } POPS_HD Real pressure(const State& value) const { return canonical.pressure(unpack(value)); } - POPS_HD State flux(const State& value, const Aux& aux, int axis) const { + POPS_HD State flux(const State& value, const auto& aux, int axis) const { return pack(canonical.flux(unpack(value), aux, axis)); } - POPS_HD Real max_wave_speed(const State& value, const Aux& aux, int axis) const { + POPS_HD Real max_wave_speed(const State& value, const auto& aux, int axis) const { return canonical.max_wave_speed(unpack(value), aux, axis); } - POPS_HD void wave_speeds(const State& value, const Aux& aux, int axis, Real& lower, + POPS_HD void wave_speeds(const State& value, const auto& aux, int axis, Real& lower, Real& upper) const { canonical.wave_speeds(unpack(value), aux, axis, lower, upper); } @@ -120,8 +120,8 @@ struct PermutedEuler { int axis) const { return pack(canonical.hllc_star_state(unpack(value), pressure_value, speed, contact, axis)); } - POPS_HD State roe_dissipation(const State& left, const Aux& left_aux, const State& right, - const Aux& right_aux, int axis) const { + POPS_HD State roe_dissipation(const State& left, const auto& left_aux, const State& right, + const auto& right_aux, int axis) const { return pack(canonical.roe_dissipation(unpack(left), left_aux, unpack(right), right_aux, axis)); } }; @@ -136,7 +136,7 @@ struct IsoHLLC { static constexpr int n_vars = 5; Real cs2 = 0.5; - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { const int in = (dir == 0) ? 1 : 2; const int it = (dir == 0) ? 2 : 1; const Real un = u[in] / u[0]; @@ -148,14 +148,14 @@ struct IsoHLLC { F[4] = u[4] * un; return F; } - POPS_HD Real max_wave_speed(const State& u, const Aux&, int dir) const { + POPS_HD Real max_wave_speed(const State& u, const auto&, int dir) const { const int in = (dir == 0) ? 1 : 2; const Real un = u[in] / u[0]; const Real c = std::sqrt(cs2); const Real a = un < 0 ? -un : un; return a + c; } - POPS_HD void wave_speeds(const State& u, const Aux&, int dir, Real& smin, Real& smax) const { + POPS_HD void wave_speeds(const State& u, const auto&, int dir, Real& smin, Real& smax) const { const int in = (dir == 0) ? 1 : 2; const Real un = u[in] / u[0]; const Real c = std::sqrt(cs2); @@ -197,7 +197,7 @@ struct DimensionalIsoHLLC { static constexpr int tracer_component = Dimension + 1; Real cs2 = Real(0.5); - POPS_HD State flux(const State& value, const Aux&, int axis) const { + POPS_HD State flux(const State& value, const auto&, int axis) const { const int normal = axis + 1; const Real normal_velocity = value[normal] / value[0]; State result{}; @@ -209,13 +209,13 @@ struct DimensionalIsoHLLC { return result; } - POPS_HD Real max_wave_speed(const State& value, const Aux&, int axis) const { + POPS_HD Real max_wave_speed(const State& value, const auto&, int axis) const { const Real normal_velocity = value[axis + 1] / value[0]; const Real absolute_velocity = normal_velocity < Real(0) ? -normal_velocity : normal_velocity; return absolute_velocity + std::sqrt(cs2); } - POPS_HD void wave_speeds(const State& value, const Aux&, int axis, Real& lower, + POPS_HD void wave_speeds(const State& value, const auto&, int axis, Real& lower, Real& upper) const { const Real normal_velocity = value[axis + 1] / value[0]; const Real sound_speed = std::sqrt(cs2); diff --git a/tests/cpp/unit/numerics/test_weno_convergence.cpp b/tests/cpp/unit/numerics/test_weno_convergence.cpp index d10b0d853..9a6b99b07 100644 --- a/tests/cpp/unit/numerics/test_weno_convergence.cpp +++ b/tests/cpp/unit/numerics/test_weno_convergence.cpp @@ -62,8 +62,8 @@ struct PrimitiveTestModel { static constexpr int n_vars = 2; int* primitive_calls = nullptr; - POPS_HD State flux(const State& state, const Aux&, int) const { return state; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(1); } + POPS_HD State flux(const State& state, const auto&, int) const { return state; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(1); } POPS_HD State source(const State&, const Aux&) const { return State{}; } POPS_HD Real elliptic_rhs(const State&) const { return Real(0); } diff --git a/tests/cpp/unit/physics/test_adaptive_multirate.cpp b/tests/cpp/unit/physics/test_adaptive_multirate.cpp index b82f8e733..14673e9ff 100644 --- a/tests/cpp/unit/physics/test_adaptive_multirate.cpp +++ b/tests/cpp/unit/physics/test_adaptive_multirate.cpp @@ -28,10 +28,10 @@ struct AdvectProduce { using Aux = pops::Aux; static constexpr int n_vars = 1; Real a = Real(1), rate = Real(1); - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { return State{dir == 0 ? a * u[0] : Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return a < 0 ? -a : a; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return a < 0 ? -a : a; } POPS_HD State source(const State&, const Aux&) const { return State{rate}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/unit/physics/test_aux_coupler_bz.cpp b/tests/cpp/unit/physics/test_aux_coupler_bz.cpp index 7bfc36cd4..01c540776 100644 --- a/tests/cpp/unit/physics/test_aux_coupler_bz.cpp +++ b/tests/cpp/unit/physics/test_aux_coupler_bz.cpp @@ -29,8 +29,8 @@ struct BzGrow { using Aux = pops::Aux; static constexpr int n_vars = 1; static constexpr int n_aux = 4; // phi, grad_x, grad_y, B_z - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State& u, const Aux& a) const { State s{}; s[0] = a.B_z * u[0]; diff --git a/tests/cpp/unit/physics/test_aux_extra.cpp b/tests/cpp/unit/physics/test_aux_extra.cpp index 080f77a59..c3a65f8eb 100644 --- a/tests/cpp/unit/physics/test_aux_extra.cpp +++ b/tests/cpp/unit/physics/test_aux_extra.cpp @@ -34,8 +34,8 @@ struct MagSource { using Aux = pops::Aux; static constexpr int n_vars = 1; static constexpr int n_aux = 4; // phi, grad_x, grad_y, B_z - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State& u, const Aux& a) const { State s{}; s[0] = a.B_z * u[0]; @@ -49,8 +49,8 @@ struct GradSource { using State = StateVec<1>; using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State& u, const Aux& a) const { State s{}; s[0] = a.grad_x * u[0]; diff --git a/tests/cpp/unit/physics/test_multirate_stride.cpp b/tests/cpp/unit/physics/test_multirate_stride.cpp index d6962c7b5..34323edf1 100644 --- a/tests/cpp/unit/physics/test_multirate_stride.cpp +++ b/tests/cpp/unit/physics/test_multirate_stride.cpp @@ -26,8 +26,8 @@ struct Production { using Aux = pops::Aux; static constexpr int n_vars = 1; Real rate = Real(1); - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{rate}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/unit/physics/test_polar_mms_vr.cpp b/tests/cpp/unit/physics/test_polar_mms_vr.cpp index da3d726bc..0a84b684f 100644 --- a/tests/cpp/unit/physics/test_polar_mms_vr.cpp +++ b/tests/cpp/unit/physics/test_polar_mms_vr.cpp @@ -130,15 +130,17 @@ struct MmsTransportPolar { static constexpr int n_aux = 4; // lit phi, grad_r, grad_theta (0..2) + S au canal extra 3 (B_z) using State = StateVec<1>; Real B0 = 1; - POPS_HD Real velocity(const Aux& a, int dir) const { - return (dir == 0) ? (-a.grad_y / B0) : (a.grad_x / B0); + POPS_HD Real velocity(const auto& providers, int dir) const { + const Real grad_x = providers.template flux_provider<1>(); + const Real grad_y = providers.template flux_provider<2>(); + return (dir == 0) ? (-grad_y / B0) : (grad_x / B0); } - POPS_HD StateVec<1> flux(const StateVec<1>& u, const Aux& a, int dir) const { + POPS_HD StateVec<1> flux(const StateVec<1>& u, const auto& a, int dir) const { StateVec<1> f{}; f[0] = u[0] * velocity(a, dir); return f; } - POPS_HD Real max_wave_speed(const StateVec<1>&, const Aux& a, int dir) const { + POPS_HD Real max_wave_speed(const StateVec<1>&, const auto& a, int dir) const { const Real d = velocity(a, dir); return d < 0 ? -d : d; } diff --git a/tests/cpp/unit/physics/test_two_species_minimal.cpp b/tests/cpp/unit/physics/test_two_species_minimal.cpp index 828846ae2..472883b4f 100644 --- a/tests/cpp/unit/physics/test_two_species_minimal.cpp +++ b/tests/cpp/unit/physics/test_two_species_minimal.cpp @@ -37,8 +37,8 @@ struct ElectronRelax { Real k = Real(1000); // raideur Real neq = Real(1); // densite d'equilibre - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State& u, const Aux&) const { return State{-k * (u[0] - neq)}; } POPS_HD Real elliptic_rhs(const State& u) const { return -u[0]; } }; @@ -51,8 +51,8 @@ struct IonProduction { Real rate = Real(3); - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{rate}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/unit/physics/test_user_time_integrator.cpp b/tests/cpp/unit/physics/test_user_time_integrator.cpp index 1cc5122ca..016e035d0 100644 --- a/tests/cpp/unit/physics/test_user_time_integrator.cpp +++ b/tests/cpp/unit/physics/test_user_time_integrator.cpp @@ -25,8 +25,8 @@ struct Production { using Aux = pops::Aux; static constexpr int n_vars = 1; Real rate = Real(3); - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{rate}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/unit/runtime/test_assembler_driver.cpp b/tests/cpp/unit/runtime/test_assembler_driver.cpp index a51e52f46..6b98168d3 100644 --- a/tests/cpp/unit/runtime/test_assembler_driver.cpp +++ b/tests/cpp/unit/runtime/test_assembler_driver.cpp @@ -32,8 +32,8 @@ struct Scalar { using State = StateVec<1>; using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/unit/runtime/test_coupled_source.cpp b/tests/cpp/unit/runtime/test_coupled_source.cpp index 391299ddd..98ca6777c 100644 --- a/tests/cpp/unit/runtime/test_coupled_source.cpp +++ b/tests/cpp/unit/runtime/test_coupled_source.cpp @@ -30,8 +30,8 @@ struct Inert { using State = StateVec<1>; using Aux = pops::Aux; static constexpr int n_vars = 1; - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } POPS_HD Real elliptic_rhs(const State& u) const { return u[0]; } }; diff --git a/tests/cpp/unit/runtime/test_disc_domain_mask.cpp b/tests/cpp/unit/runtime/test_disc_domain_mask.cpp index 184919f46..aed5ed3be 100644 --- a/tests/cpp/unit/runtime/test_disc_domain_mask.cpp +++ b/tests/cpp/unit/runtime/test_disc_domain_mask.cpp @@ -49,10 +49,10 @@ struct Advect { using Aux = pops::Aux; static constexpr int n_vars = 1; Real vx = 0.0, vy = 0.0; - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { return State{(dir == 0 ? vx : vy) * u[0]}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int dir) const { + POPS_HD Real max_wave_speed(const State&, const auto&, int dir) const { return std::fabs(dir == 0 ? vx : vy); } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } @@ -62,7 +62,7 @@ struct Advect { // Device-side Roe providers cannot throw. This model is the exact failure carrier emitted by a // dense-Jacobian Roe provider when its eigensolve reports a complex or unresolved spectrum. struct FailedRoeAdvect : Advect { - POPS_HD State roe_dissipation(const State&, const Aux&, const State&, const Aux&, int) const { + POPS_HD State roe_dissipation(const State&, const auto&, const State&, const auto&, int) const { return State{std::numeric_limits::quiet_NaN()}; } }; diff --git a/tests/cpp/unit/runtime/test_eb_transport.cpp b/tests/cpp/unit/runtime/test_eb_transport.cpp index cc15b334c..297111112 100644 --- a/tests/cpp/unit/runtime/test_eb_transport.cpp +++ b/tests/cpp/unit/runtime/test_eb_transport.cpp @@ -63,10 +63,10 @@ struct Advect { using Aux = pops::Aux; static constexpr int n_vars = 1; Real vx = 0.0, vy = 0.0; - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { return State{(dir == 0 ? vx : vy) * u[0]}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int dir) const { + POPS_HD Real max_wave_speed(const State&, const auto&, int dir) const { return std::fabs(dir == 0 ? vx : vy); } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } @@ -74,7 +74,7 @@ struct Advect { }; struct FailedRoeAdvect : Advect { - POPS_HD State roe_dissipation(const State&, const Aux&, const State&, const Aux&, int) const { + POPS_HD State roe_dissipation(const State&, const auto&, const State&, const auto&, int) const { return State{std::numeric_limits::quiet_NaN()}; } }; diff --git a/tests/cpp/unit/runtime/test_embedded_boundary_generic.cpp b/tests/cpp/unit/runtime/test_embedded_boundary_generic.cpp index 993616965..85c941686 100644 --- a/tests/cpp/unit/runtime/test_embedded_boundary_generic.cpp +++ b/tests/cpp/unit/runtime/test_embedded_boundary_generic.cpp @@ -68,10 +68,10 @@ struct Advect { using Aux = pops::Aux; static constexpr int n_vars = 1; Real vx = 0.0, vy = 0.0; - POPS_HD State flux(const State& u, const Aux&, int dir) const { + POPS_HD State flux(const State& u, const auto&, int dir) const { return State{(dir == 0 ? vx : vy) * u[0]}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int dir) const { + POPS_HD Real max_wave_speed(const State&, const auto&, int dir) const { return std::fabs(dir == 0 ? vx : vy); } POPS_HD State source(const State&, const Aux&) const { return State{Real(0)}; } diff --git a/tests/gpu/romeo/gpu_aux_validate.cpp b/tests/gpu/romeo/gpu_aux_validate.cpp index 857c95beb..7b6434bdb 100644 --- a/tests/gpu/romeo/gpu_aux_validate.cpp +++ b/tests/gpu/romeo/gpu_aux_validate.cpp @@ -49,8 +49,8 @@ struct TeProbe { using Aux = pops::Aux; static constexpr int n_vars = 1; static constexpr int n_aux = 5; // phi, grad_x, grad_y, B_z, T_e - POPS_HD State flux(const State&, const Aux&, int) const { return State{Real(0)}; } - POPS_HD Real max_wave_speed(const State&, const Aux&, int) const { return Real(0); } + POPS_HD State flux(const State&, const auto&, int) const { return State{Real(0)}; } + POPS_HD Real max_wave_speed(const State&, const auto&, int) const { return Real(0); } POPS_HD State source(const State& u, const Aux& a) const { State s{}; s[0] = a.T_e * u[0]; // lit la composante aux 4 (T_e) From c731100d24eb0fc7d66d5f8885e710cebaa203ab Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 07:08:43 +0200 Subject: [PATCH 107/109] fix(release): reconcile merged service contracts --- include/pops/runtime/program/program_context.hpp | 6 +++--- .../runtime/program/program_execution_services.hpp | 13 +++++-------- .../test_async_scientific_output_diagnostics.py | 1 + 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/include/pops/runtime/program/program_context.hpp b/include/pops/runtime/program/program_context.hpp index fab5a6518..ca143a418 100644 --- a/include/pops/runtime/program/program_context.hpp +++ b/include/pops/runtime/program/program_context.hpp @@ -668,7 +668,7 @@ class ProgramContext : public ProgramExecutionServices { // terms remain available and the future selector must fail closed on this missing producer. if (sys_->program_is_polar()) return std::nullopt; - const GridContext context = program_execution_block_grid_context_(program_block); + const GridContext context = sys_->grid_context(sys_block(program_block)); const Real cell_measure = context.geom.dx() * context.geom.dy(); if (!std::isfinite(static_cast(cell_measure)) || cell_measure <= Real(0)) throw std::runtime_error( @@ -805,8 +805,8 @@ class ProgramContext : public ProgramExecutionServices { const HistoryRegistration& registration) const { return sys_->history_initialized(registration.name); } - double program_execution_history_slot_dt_storage_( - const HistoryRegistration& registration, int lag) const { + double program_execution_history_slot_dt_storage_(const HistoryRegistration& registration, + int lag) const { return sys_->history_slot_dt(registration.name, lag); } void program_execution_set_history_initialized_storage_(const HistoryRegistration& registration, diff --git a/include/pops/runtime/program/program_execution_services.hpp b/include/pops/runtime/program/program_execution_services.hpp index 33686ed63..86e69aefb 100644 --- a/include/pops/runtime/program/program_execution_services.hpp +++ b/include/pops/runtime/program/program_execution_services.hpp @@ -519,12 +519,12 @@ class ProgramExecutionServices { /// provider also supplies exact metric-integrated component values before and after projection; /// their signed delta stays qualified by runtime block/level/component in the attempt mailbox. void apply_projection(int block, MultiFab& state) const { - const int runtime_block = sys_block(block); ProgramRuntimeState& runtime = program_runtime_state_(); if (!runtime.automatic_balance_capture_due()) { - provider_().program_execution_apply_projection_(runtime_block, state); + provider_().program_execution_apply_projection_(sys_block(block), state); return; } + const int runtime_block = sys_block(block); const std::optional> before = provider_().program_execution_projection_balance_integrals_(block, state); provider_().program_execution_apply_projection_(runtime_block, state); @@ -918,8 +918,7 @@ class ProgramExecutionServices { if (!std::isfinite(static_cast(target_offset))) throw std::invalid_argument("linear history interpolation offset must be finite"); - HistoryRegistration registration = - history_registration_(name, max_lag, /*ncomp=*/-1, owner); + HistoryRegistration registration = history_registration_(name, max_lag, /*ncomp=*/-1, owner); if (!provider_().program_execution_history_initialized_storage_(registration)) throw std::runtime_error( "linear history interpolation requires an initialized native history"); @@ -963,13 +962,11 @@ class ProgramExecutionServices { const double logical_fraction = coordinate + static_cast(older_lag); const double target_time = older_time + logical_fraction * bracket_dt; const double timestamp_fraction = (target_time - older_time) / (newer_time - older_time); - if (!std::isfinite(timestamp_fraction) || timestamp_fraction < 0.0 || - timestamp_fraction > 1.0) + if (!std::isfinite(timestamp_fraction) || timestamp_fraction < 0.0 || timestamp_fraction > 1.0) throw std::runtime_error( "linear history interpolation target does not bracket native timestamps"); - registration = - ensure_history_registered_(name, older_lag, /*ncomp=*/-1, owner); + registration = ensure_history_registered_(name, older_lag, /*ncomp=*/-1, owner); MultiFab& older = provider_().program_execution_read_history_storage_( registration, older_lag, HistoryReadMode::RequireInitialized); MultiFab& newer = provider_().program_execution_read_history_storage_( diff --git a/tests/python/unit/output/test_async_scientific_output_diagnostics.py b/tests/python/unit/output/test_async_scientific_output_diagnostics.py index 7fb6b856a..66be0b765 100644 --- a/tests/python/unit/output/test_async_scientific_output_diagnostics.py +++ b/tests/python/unit/output/test_async_scientific_output_diagnostics.py @@ -100,6 +100,7 @@ def test_async_scientific_output_accepts_diagnostic_only_and_resolves_balance(): "reduction": "accepted_balance", "transform": "identity", "metric_weighted": False, + "coefficient": (1.0).hex(), "balance_route": ledger.route_identity(block).token, }, ) From 05a3b1f4e273b8356e754c475d17b13a976ff059 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 07:22:53 +0200 Subject: [PATCH 108/109] test(history): reduce the explicit current state --- tests/python/integration/io/test_amr_history_checkpoint.py | 2 +- .../integration/io/test_uniform_selective_history_checkpoint.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python/integration/io/test_amr_history_checkpoint.py b/tests/python/integration/io/test_amr_history_checkpoint.py index a8fde2d81..7b6d0e69b 100644 --- a/tests/python/integration/io/test_amr_history_checkpoint.py +++ b/tests/python/integration/io/test_amr_history_checkpoint.py @@ -159,7 +159,7 @@ def _state3_program( BalanceDueRoute, ) - total = P.sum(U) + total = P.sum(U.n) ledger = BalanceLedger("amr-selective-replay") P.record_balance( ledger, diff --git a/tests/python/integration/io/test_uniform_selective_history_checkpoint.py b/tests/python/integration/io/test_uniform_selective_history_checkpoint.py index 98b3e7a36..03c54d462 100644 --- a/tests/python/integration/io/test_uniform_selective_history_checkpoint.py +++ b/tests/python/integration/io/test_uniform_selective_history_checkpoint.py @@ -91,7 +91,7 @@ def _program(model): _case, states = program_states(program, model, ("blk",)) state = states["blk"] program.keep_history(state, depth=4, checkpoint_policy=Interval(2)) - total = program.sum(state) + total = program.sum(state.n) ledger = BalanceLedger("uniform-selective-replay") program.record_balance( ledger, From 7a51a2000eddc1c1edd1469d352c9924aa38f980 Mon Sep 17 00:00:00 2001 From: desp0042 Date: Sun, 2 Aug 2026 07:23:00 +0200 Subject: [PATCH 109/109] test(runtime): pass exact balance owner coordinates --- .../runtime/test_runtime_instance_gate.py | 26 ++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/tests/python/unit/runtime/test_runtime_instance_gate.py b/tests/python/unit/runtime/test_runtime_instance_gate.py index bbf034baa..f04087038 100644 --- a/tests/python/unit/runtime/test_runtime_instance_gate.py +++ b/tests/python/unit/runtime/test_runtime_instance_gate.py @@ -1932,7 +1932,13 @@ def _accepted_balance_terms(self, route): } terms = RuntimeConsumerPublisher._native_balance_terms( - _Provider(), "pops.balance-ledger-route.v1:sha256:" + "1" * 64) + _Provider(), + "pops.balance-ledger-route.v1:sha256:" + "1" * 64, + block="fluid", + component=0, + levels=(0,), + automatic_terms=(), + ) assert terms.residual == 4.0 assert terms.reflux == 3.0 @@ -1941,7 +1947,14 @@ def _accepted_balance_terms(self, _route): return {"storage_change": 1.0} with pytest.raises(TypeError, match="exactly storage_change"): - RuntimeConsumerPublisher._native_balance_terms(_Incomplete(), "route") + RuntimeConsumerPublisher._native_balance_terms( + _Incomplete(), + "route", + block="fluid", + component=0, + levels=(0,), + automatic_terms=(), + ) class _Coerced: def _accepted_balance_terms(self, _route): @@ -1954,7 +1967,14 @@ def _accepted_balance_terms(self, _route): } with pytest.raises(TypeError, match="exact floating-point"): - RuntimeConsumerPublisher._native_balance_terms(_Coerced(), "route") + RuntimeConsumerPublisher._native_balance_terms( + _Coerced(), + "route", + block="fluid", + component=0, + levels=(0,), + automatic_terms=(), + ) def test_diagnostic_restart_restores_payload_terms_and_native_inspection_registry():