From 391da963c23cd7746d44e7f5ea72173eac038c5e Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 14:39:18 -0700 Subject: [PATCH 01/14] Load config.default.yaml as the base layer; sparsify user configs config.default.yaml documented every user-facing option but was never loaded, so every scenario config (tutorial, test, equivalence) had to be a full copy of it and drifted structurally. config.test.yaml had drifted far enough that it no longer carried `solving:` or `costs.atb`, which cluster_simpl and build_cost_data dereference directly. Snakefile now loads repo_data/config/config.default.yaml as the last layered configfile, beneath whatever the user passes with --configfile, and config.tutorial.yaml / config.test.yaml become sparse overlays carrying only the keys that differ from the merged base (plus scenario.interconnect, which is pinned deliberately so the footprint cannot follow a change to the default scenario). Lists are replaced wholesale by snakemake's merge, so any differing list is restated in full. The two equivalence configs stay self-contained on purpose: the harness copies them into an anchor worktree (upstream e7f8bd70) that does not load config.default.yaml, so a sparse overlay there would feed the two sides different configs. GURO_PAR_BARDENSETHRESH is added to their gurobi-default block for the same reason - the candidate now inherits it from the default layer and the anchor never would. Verified by replaying snakemake's update_config merge for all five configs against the origin/develop baseline: no key changes value; the only diffs are additions from the default layer, plus model_topology.include/aggregate going from {} to None (update_config maps an empty dict onto a null parent as null). Every consumer of those two keys is a falsy check or an `is not None` guard that skips the same work either way. Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 21 +- .../config/config.equivalence-usa.yaml | 10 + .../repo_data/config/config.equivalence.yaml | 10 + workflow/repo_data/config/config.test.yaml | 105 ++---- .../repo_data/config/config.tutorial.yaml | 337 ++---------------- 5 files changed, 88 insertions(+), 395 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 809f06e62..b7b034041 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -81,15 +81,28 @@ wildcard_constraints: # -------------------------- Config and Subworkflows ------------------------- # -# Merge subworkflow configs and main config +# Layered configuration. Snakemake merges each `configfile:` into `config` +# with a recursive dict update (nested mappings merge; lists and scalars are +# REPLACED wholesale), and anything the user passes via `--configfile` is +# merged last, on top of everything below. +# +# config.cluster.yaml HPC scheduler settings +# config.common.yaml renewable/atlite plumbing that rarely changes +# config.plotting.yaml figure styling +# config.api.yaml API keys (gitignored) +# config.sector.yaml sector-coupling defaults +# config.default.yaml the canonical scenario base: every user-facing knob +# +# config.default.yaml is loaded from the tracked repo_data/ templates rather +# than the per-user config/ copy: it is the workflow's base layer, not a +# user-owned file. Users copy it to config/config..yaml and pass +# that with --configfile, which then only needs to carry the keys it changes. configfile: "config/config.cluster.yaml" configfile: "config/config.common.yaml" configfile: "config/config.plotting.yaml" configfile: "config/config.api.yaml" configfile: "config/config.sector.yaml" - - -# configfile: "config/config.default.yaml" +configfile: "repo_data/config/config.default.yaml" run = config.get("run", {}) diff --git a/workflow/repo_data/config/config.equivalence-usa.yaml b/workflow/repo_data/config/config.equivalence-usa.yaml index 39777bb59..6d810db53 100644 --- a/workflow/repo_data/config/config.equivalence-usa.yaml +++ b/workflow/repo_data/config/config.equivalence-usa.yaml @@ -10,6 +10,11 @@ # Keys the ANCHOR requires from the main config (its layered common lacks # them): co2.storage, clustering.cluster_network.feature, # clustering.simplify_network.feature (prong 2). v1-epic ignores the extras. +# +# DELIBERATELY NOT A SPARSE OVERLAY. Unlike config.tutorial.yaml and +# config.test.yaml, this file must stay self-contained: the anchor checkout +# does NOT auto-load config.default.yaml, so any key dropped here would be +# supplied to the candidate and missing on the anchor. run: name: "equivalence" disable_progressbar: true @@ -207,6 +212,11 @@ solving: Seed: 123 AggFill: 0 PreDual: 0 + # Mirrors config.default.yaml. Required explicitly here: the candidate + # branch now auto-loads config.default.yaml as a base layer, so omitting + # this would hand the candidate a solver option the anchor never sees and + # break the like-for-like comparison. + GURO_PAR_BARDENSETHRESH: 200 focus_weights: diff --git a/workflow/repo_data/config/config.equivalence.yaml b/workflow/repo_data/config/config.equivalence.yaml index a86b4b127..0c95964d5 100644 --- a/workflow/repo_data/config/config.equivalence.yaml +++ b/workflow/repo_data/config/config.equivalence.yaml @@ -8,6 +8,11 @@ # Keys the ANCHOR requires from the main config (its layered common lacks # them): co2.storage, clustering.cluster_network.feature, # clustering.simplify_network.feature (prong 2). v1-epic ignores the extras. +# +# DELIBERATELY NOT A SPARSE OVERLAY. Unlike config.tutorial.yaml and +# config.test.yaml, this file must stay self-contained: the anchor checkout +# does NOT auto-load config.default.yaml, so any key dropped here would be +# supplied to the candidate and missing on the anchor. run: name: "equivalence" disable_progressbar: true @@ -206,6 +211,11 @@ solving: Seed: 123 AggFill: 0 PreDual: 0 + # Mirrors config.default.yaml. Required explicitly here: the candidate + # branch now auto-loads config.default.yaml as a base layer, so omitting + # this would hand the candidate a solver option the anchor never sees and + # break the like-for-like comparison. + GURO_PAR_BARDENSETHRESH: 200 focus_weights: diff --git a/workflow/repo_data/config/config.test.yaml b/workflow/repo_data/config/config.test.yaml index 42abac612..986aff79f 100644 --- a/workflow/repo_data/config/config.test.yaml +++ b/workflow/repo_data/config/config.test.yaml @@ -1,37 +1,36 @@ -# PyPSA-USA test config — used by tests/integration to build to -# cluster_network on a minimal CA-only Western slice in <5 minutes. -# DO NOT use as a user-facing entry point. +# ==================================================================== +# PyPSA-USA — Test Configuration (Tier B integration harness) +# ==================================================================== +# Builds to `cluster_network` on a minimal CA-only Western slice in +# <5 minutes. DO NOT use as a user-facing entry point. +# +# This is a SPARSE OVERLAY. `workflow/Snakefile` already loads the +# layered base (config.cluster / config.common / config.plotting / +# config.api / config.sector / config.default) underneath it, so only +# keys that DIFFER from that merged base belong here. +# +# `scenario.interconnect` is pinned deliberately even though it happens +# to match config.default.yaml: the Tier B time budget depends on it, +# and it must not follow a future change to the default scenario. +# +# NB: lists are REPLACED wholesale by the merge, never merged element- +# wise, so any list that differs must be restated in full. +# ==================================================================== + run: name: "test" disable_progressbar: true - shared_resources: false - shared_cutouts: true - validation: false - -foresight: 'perfect' scenario: - interconnect: [western] + interconnect: [western] # pinned: Tier B budget depends on the footprint clusters: [4m] simpl: [20] - opts: [REM-3h] - ll: [v1.0] scope: "total" - sector: "" planning_horizons: [2030] model_topology: - transmission_network: 'reeds' - topological_boundaries: 'reeds_zone' - interface_transmission_limits: false include: reeds_state: ['CA'] - aggregate: {} - -enable: - build_cutout: false - -renewable_weather_years: [2019] snapshots: start: "2019-01-01 00:00" @@ -39,71 +38,7 @@ snapshots: inclusive: both electricity: - conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, geothermal, biomass, waste] renewable_carriers: [onwind, solar, hydro] - retirement: economic extendable_carriers: Generator: [solar, onwind, OCGT, CCGT] StorageUnit: [4hr_battery_storage] - Store: [] - Link: [] - demand: - profile: efs - scenario: - efs_case: reference - efs_speed: moderate - aeo: reference - -conventional: - unit_commitment: false - must_run: false - dynamic_fuel_price: - enable: false - pudl: true - wholesale: true - -lines: - s_max_pu: 0.7 - s_nom_max: .inf - max_extension: 20000 - length_factor: 1.25 - -links: - p_max_pu: 1.0 - p_nom_max: .inf - max_extension: 20000 - -costs: - ng_fuel_year: 2019 - -clustering: - simplify_network: - algorithm: kmeans - cluster_network: - algorithm: kmeans - exclude_carriers: [] - consider_efficiency_classes: false - aggregation_strategies: - generators: - build_year: 'capacity_weighted_average' - lifetime: 'capacity_weighted_average' - start_up_cost: 'capacity_weighted_average' - min_up_time: 'capacity_weighted_average' - min_down_time: 'capacity_weighted_average' - ramp_limit_up: max - ramp_limit_down: max - committable: any - vom_cost: mean - fuel_cost: mean - heat_rate: mean - temporal: - resolution_elec: false - resolution_sector: false - -focus_weights: - -custom_files: - activate: false - files_path: '' - network_name: '' - diff --git a/workflow/repo_data/config/config.tutorial.yaml b/workflow/repo_data/config/config.tutorial.yaml index 2ed7e3ebb..54a998939 100644 --- a/workflow/repo_data/config/config.tutorial.yaml +++ b/workflow/repo_data/config/config.tutorial.yaml @@ -2,18 +2,22 @@ # PyPSA-USA — Tutorial Configuration (minimal smoke-test run) # ==================================================================== # California-only, 4-cluster, single 2050 horizon — the smallest -# meaningful end-to-end run. Use this to confirm a fresh checkout -# can build, cluster, and solve a network. +# meaningful end-to-end run. Use this to confirm a fresh checkout can +# build, cluster, and solve a network. # # Audience: new users verifying their install. Run with: # cd workflow -# uv run snakemake -j1 --configfile config/config.tutorial.yaml +# uv run snakemake -j1 --configfile repo_data/config/config.tutorial.yaml # -# Intentionally omitted (silent defaults from config.common.yaml or -# script defaults apply): co2, dac, electricity.erm, -# electricity.demand_response, electricity.imports, electricity.exports, -# walltime, renewable_scenarios, renewable_snapshots, costs.min_year, -# costs.max_growth. See config.default.yaml for the full surface. +# This is a SPARSE OVERLAY. `workflow/Snakefile` already loads the +# layered base (config.cluster / config.common / config.plotting / +# config.api / config.sector / config.default) underneath it, so only +# keys that DIFFER from that merged base belong here. Everything not +# listed — solver options, costs, clustering strategies, lines/links, +# conventional generators, CO2, DAC — comes from config.default.yaml. +# +# NB: lists are REPLACED wholesale by the merge, never merged element- +# wise, so any list that differs must be restated in full. # # Option reference: docs/source/config-configuration.md # ==================================================================== @@ -22,204 +26,46 @@ # RUN — run identity # ==================================================================== run: - name: "Tutorial" # label that becomes the resources// subdir - disable_progressbar: false # silence atlite/snakemake progress bars - shared_resources: false # false = isolate resources/ per run.name - shared_cutouts: true # true = share atlite cutouts across runs (recommended) - validation: false # back-casting validation plots (historical-only) + name: "Tutorial" # resources// subdir for this run # ==================================================================== # SCENARIO — workflow wildcards and planning horizons # ==================================================================== -# docs : SCENARIO scenario: - interconnect: [western] # geographic scope; usa | texas | western | eastern - planning_horizons: [2050] # investment-period years - clusters: [4m] # final cluster count; integer optionally suffixed m/a/c, or "all" - simpl: [75] # pre-clustering kmeans granularity - ll: [v1.0] # line-limit scenario; v|c + number/opt/all - opts: [REM-3h] # options string; REM = renewable-energy mix, 3h = 3-hour resolution + interconnect: [western] # pinned: the tutorial is a CA slice of the Western Interconnect + planning_horizons: [2050] # single horizon keeps the solve small + clusters: [4m] # 4 clusters, m = memory-scaled resource hint scope: "total" # demand scope; urban | rural | total - sector: "" # sector coupling; "" (electricity-only) | E | G | E-G - -foresight: 'perfect' # perfect (monolithic) | myopic (sequential) # ==================================================================== -# MODEL TOPOLOGY — transmission backbone and zonal aggregation +# MODEL TOPOLOGY — restrict the footprint to California # ==================================================================== model_topology: - transmission_network: 'reeds' # reeds | tamu - topological_boundaries: 'reeds_zone' # county | reeds_zone (tutorial pins to reeds_zone) - interface_transmission_limits: false # NARIS2024 inter-region transfer caps - include: # restrict to California REeDS zones - # reeds_zone: [] - reeds_state: ['CA'] - # reeds_ba: [] - aggregate: # eligible keys: reeds_zone | trans_reg - # trans_grp: [] - # reeds_zone: [] - - -# ==================================================================== -# ENABLE — top-level feature flags -# ==================================================================== -# docs : ENABLE -enable: - build_cutout: false # false = use prebuilt atlite cutout - - -# ==================================================================== -# SNAPSHOTS — temporal scope -# ==================================================================== -renewable_weather_years: [2019] # weather year sourced from common.yaml cutout - -snapshots: - start: "2019-01-01" - end: "2020-01-01" - inclusive: "left" # pandas.date_range inclusive arg + include: + reeds_state: ['CA'] # restrict the network to California REeDS zones # ==================================================================== -# ELECTRICITY — generators, storage, demand, reserves +# ELECTRICITY — expansion candidates # ==================================================================== -# docs : ELECTRICITY +# Same as config.default.yaml minus `hydrogen_ct` (whose `costs.min_year` +# of 2040 makes it irrelevant to this single-horizon tutorial). electricity: - conventional_carriers: [nuclear, oil, OCGT, CCGT, coal, geothermal, biomass, waste] - renewable_carriers: [onwind, offwind_floating, solar, hydro] - retirement: economic # economic | technical + extendable_carriers: + Generator: [solar, onwind, offwind_floating, OCGT, CCGT, CCGT-95CCS, coal, nuclear] - SAFE_reservemargin: 0.14 # SAFE PRM (fraction of peak load) - regional_Co2_limits: 'config/policy_constraints/regional_Co2_limits.csv' - technology_capacity_targets: 'config/policy_constraints/technology_capacity_targets.csv' - portfolio_standards: 'config/policy_constraints/portfolio_standards.csv' - SAFE_regional_reservemargins: 'config/policy_constraints/SAFE_regional_prm.csv' - transmission_interface_limits: 'config/policy_constraints/transmission_interface_limits.csv' - operational_reserve: # GenX-style reserves; only active when activate=true - activate: false - epsilon_load: 0.02 # share of total load held as reserve - epsilon_vres: 0.02 # share of VRE supply held as reserve - contingency: 4000 # fixed reserve floor (MW) - - extendable_carriers: # candidates for capacity expansion - Generator: [solar, onwind, offwind_floating, OCGT, CCGT, CCGT-95CCS, coal, nuclear] - StorageUnit: [4hr_battery_storage, 8hr_battery_storage] # Xhr_battery_storage (X = 2..10) - Store: [] - Link: [] - - demand: - profile: efs # efs (EIA-EFS) | eia (historical actuals) - scenario: # EFS scenario controls - efs_case: reference # reference | medium | high - efs_speed: moderate # slow | moderate | rapid - aeo: reference # AEO scaling case - - -# ==================================================================== -# CONVENTIONAL — unit commitment and fuel-price overrides -# ==================================================================== -# docs : CONVENTIONAL -conventional: - unit_commitment: false # enforce min up/down + startup costs - must_run: false # ADS must-run minimum loading on flagged thermal plants - dynamic_fuel_price: - enable: false # true = time-varying fuel prices - pudl: true # PUDL receipts-based monthly fuel costs - wholesale: true # CAISO wholesale NG prices (overrides PUDL for NG) - - -# ==================================================================== -# LINES — AC transmission lines # ==================================================================== -# docs : LINES -lines: - s_max_pu: 0.7 # apparent-power utilization cap (0.7 = N-1 derate) - s_nom_max: .inf # MVA cap per corridor (.inf = unbounded) - max_extension: 20000 # MW cap on network-wide expansion - length_factor: 1.25 # geographic-length multiplier - - -# ==================================================================== -# LINKS — HVDC and controllable links -# ==================================================================== -# docs : LINKS -links: - p_max_pu: 1.0 # active-power utilization cap - p_nom_max: .inf # MW cap per corridor - max_extension: 20000 # MW cap on network-wide expansion - - +# SECTOR — sector-coupling settings (inert while scenario.sector is "") # ==================================================================== -# CO2 — sequestration & pipeline transport (kept disabled for tutorial) -# ==================================================================== -# Required to be present even when disabled — build_electricity.smk -# reads `co2.storage` unconditionally when assembling the DAG. -# docs : CO2 -co2: - storage: false - network: - enable: false - - -# ==================================================================== -# DAC — Direct Air Capture (kept disabled for tutorial) -# ==================================================================== -# docs : DAC -dac: - enable: false - - -# ==================================================================== -# COSTS — capex/opex scenarios and policy incentives -# ==================================================================== -# docs : COSTS -costs: - atb: - model_case: "Market" # Market | R&D - scenario: "Moderate" # Advanced | Conservative | Moderate - aeo: - scenario: "reference" # reference | high | low - social_discount_rate: 0.02 # fraction - ng_fuel_year: 2019 # CAISO NG price vintage year; 2019..2023 - emission_prices: # CO2/SOx/NOx adder (only with Ep opt) - enable: false - co2: 0. # USD/tCO2 - co2_monthly_prices: false # monthly time series instead of flat - ptc_modifier: # Production Tax Credit ($/MWh) - onwind: 27.50 - biomass: 27.50 - itc_modifier: # Investment Tax Credit (fraction of capex) - solar: 0.3 - offwind: 0.3 - offwind_floating: 0.3 - EGS: 0.3 - geothermal: 0.3 - SMR: 0.3 - nuclear: 0.3 - hydro: 0.3 - 2hr_battery_storage: 0.3 - 4hr_battery_storage: 0.3 - 6hr_battery_storage: 0.3 - 8hr_battery_storage: 0.3 - 10hr_battery_storage: 0.3 - 8hr_PHS: 0.3 # PHS = pumped hydro storage - 10hr_PHS: 0.3 - 12hr_PHS: 0.3 - max_growth: # per-carrier annual build-rate caps; {carrier: {base: MW, rate: fraction}} - - -# ==================================================================== -# SECTOR — sector-coupling settings (only consumed when scenario.sector != "") -# ==================================================================== -# docs : SECTOR +# Kept as a worked example of the sector knobs a coupled run needs; the +# remaining sector defaults come from config.sector.yaml. sector: - co2_sequestration_potential: 0 # tCO2/yr nationwide cap on sequestration when sector model is on + co2_sequestration_potential: 0 # tCO2/yr nationwide sequestration cap natural_gas: - cyclic_storage: false # true = enforce SOC continuity at year boundary - heating: - heat_pump_sink_T: 55. # heat-pump output temperature (°C) + cyclic_storage: false # false = do not enforce SOC continuity at the year boundary demand: profile: # demand profile source per end-use residential: eulp # efs (EIA-EFS) | eulp (NREL ResStock/ComStock End-Use Load Profiles) @@ -241,128 +87,7 @@ sector: # ==================================================================== -# CLUSTERING — pre-cluster (simpl) and final (cluster) settings +# SOLVING — scheduler memory hint (solver settings come from the base) # ==================================================================== -# docs : CLUSTERING -clustering: - simplify_network: - algorithm: kmeans # kmeans | modularity - cluster_network: - algorithm: kmeans # kmeans | modularity - exclude_carriers: [] # carriers dropped before clustering - consider_efficiency_classes: false # bin generators by efficiency before aggregation - aggregation_strategies: - generators: - build_year: 'capacity_weighted_average' - lifetime: 'capacity_weighted_average' - start_up_cost: 'capacity_weighted_average' - min_up_time: 'capacity_weighted_average' - min_down_time: 'capacity_weighted_average' - ramp_limit_up: max - ramp_limit_down: max - committable: any - vom_cost: mean - fuel_cost: mean - heat_rate: mean - temporal: - resolution_elec: false # int H or false = native hourly - resolution_sector: false # int H or false = native hourly - -focus_weights: # OPTIONAL: per-region weights for clustering objective - - -# ==================================================================== -# SOLVING — solver selection and options -# ==================================================================== -# docs : SOLVING solving: - # tmpdir: "path/to/tmp" # OPTIONAL: solver scratch directory - - options: - load_shedding: false # true = add high-cost slack generators (debug infeasibility) - clip_p_max_pu: 1.e-2 # threshold below which renewable p_max_pu → 0 - noisy_costs: true # break LP degeneracies with small cost noise - skip_iterations: true # skip line-iteration loop - rolling_horizon: false # solve in time-overlapping windows - seed: 123 # RNG seed for noisy_costs - track_iterations: false - min_iterations: 4 - max_iterations: 6 - transmission_losses: 2 # 0 = none; N = N-piecewise quadratic approximation - linearized_unit_commitment: true # LP relaxation of UC binaries - horizon: 8760 # rolling-horizon window length (hours) - assign_all_duals: true # populate dual variables for every constraint - - solver: - name: gurobi # gurobi | highs | cplex | cbc | glpk - options: gurobi-default # name of options block below - - solver_options: - highs-default: - # https://ergo-code.github.io/HiGHS/options/definitions.html - threads: 4 - solver: "ipm" - run_crossover: "off" - small_matrix_value: 1e-6 - large_matrix_value: 1e9 - primal_feasibility_tolerance: 1e-5 - dual_feasibility_tolerance: 1e-5 - ipm_optimality_tolerance: 1e-4 - parallel: "on" - random_seed: 123 - gurobi-default: - threads: 8 - method: 2 # 2 = barrier - crossover: 0 # 0 = barrier-only (no crossover) - BarHomogeneous: 1 # use homogeneous barrier if standard does not converge - BarConvTol: 1.e-5 - OptimalityTol: 1.e-4 - FeasibilityTol: 1.e-3 - ScaleFlag: 1 - Seed: 123 - AggFill: 0 - PreDual: 0 - GURO_PAR_BARDENSETHRESH: 200 - gurobi-numeric-focus: - name: gurobi - NumericFocus: 3 # favor stability over speed - method: 2 - crossover: 0 - BarHomogeneous: 1 - BarConvTol: 1.e-5 - FeasibilityTol: 1.e-4 - OptimalityTol: 1.e-4 - ObjScale: -0.5 - threads: 8 - Seed: 123 - gurobi-fallback: # retried after a numerical failure - name: gurobi - crossover: 0 - method: 2 - BarHomogeneous: 1 - BarConvTol: 1.e-5 - FeasibilityTol: 1.e-5 - OptimalityTol: 1.e-5 - Seed: 123 - threads: 8 - cplex-default: - threads: 4 - lpmethod: 4 # 4 = barrier - solutiontype: 2 # 2 = non-basic (no crossover) - barrier.convergetol: 1.e-5 - feasopt.tolerance: 1.e-6 - cbc-default: {} # used in CI - glpk-default: {} # used in CI - - mem: 30000 # MB of RAM hint to the SLURM scheduler; ~20 GB for 50-bus+B+I+H2, ~100 GB for 181-bus+B+I+H2 - walltime: "12:00:00" # solver wall-time cap (HH:MM:SS) - - -# ==================================================================== -# CUSTOM FILES — bring-your-own network or costs -# ==================================================================== -# docs : CUSTOM_FILES -custom_files: - activate: false # true = use the paths below instead of pipeline outputs - files_path: '' # directory containing custom .nc / costs.csv - network_name: '' # .nc filename inside files_path + mem: 30000 # MB of RAM hint for the SLURM scheduler From 25a64e2e2e8b49f37051f3ede6a411e19ad3a419 Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 14:45:59 -0700 Subject: [PATCH 02/14] Load the layered config base from repo_data; shrink init to user-owned files init_pypsa_usa.sh copied the whole template tree into the gitignored workflow/config/ once and refused to refresh, so every user's layered base (common, plotting, sector, cluster) silently froze at whatever the checkout looked like on the day they ran it. Only API keys and HPC account settings are genuinely per-user. The Snakefile now reads its entire layered base out of the tracked repo_data/config/ templates, and treats config/config.api.yaml and config/config.cluster.yaml as OPTIONAL overlays on top of tracked placeholders - a fresh clone resolves without running init at all. The policy_constraints/ CSVs move the same way: configs, rules and the opts registry now reference repo_data/config/policy_constraints/ so nothing has to be copied. init_pypsa_usa.sh now seeds three files (config.default.yaml as a scenario starting point, config.api.yaml, config.cluster.yaml), is idempotent per-file, and says what it is not copying and why. .gitignore's bare `config/` pattern also matched workflow/repo_data/config, forcing every template edit through `git add -f`; it is now scoped to /workflow/config/*. test_config_tree_sync asserted a copy relationship that no longer exists. It is replaced by two tests that guard the new invariant instead: init may only seed known per-user files, and no `configfile:` may point back into config/. Tier A/B and the Tier C harness are updated for the new paths; the harness keeps feeding the anchor worktree from config/, since that pinned upstream checkout still reads its configs from there. Merged-config diff vs origin/develop: only the seven policy_constraints paths, all repointed from config/ to repo_data/config/ where the tracked CSVs actually live. Co-Authored-By: Claude Fable 5 --- .gitignore | 9 ++- CLAUDE.md | 13 ++-- README.md | 2 +- docs/source/about-install.md | 14 +++- docs/source/about-introduction.md | 6 +- docs/source/about-usage.md | 10 ++- docs/source/data-policies.md | 6 +- docs/source/release-notes.md | 6 +- init_pypsa_usa.sh | 49 ++++++++++--- tests/docs/test_docs_config.py | 73 ++++++++++++------- tests/equivalence/build.py | 16 +++- tests/equivalence/paths.py | 10 ++- tests/integration/conftest.py | 35 +-------- tests/static/test_config_keys.py | 25 +++---- tests/static/test_dag_dryrun.py | 42 +++-------- workflow/Snakefile | 36 ++++++--- workflow/repo_data/config/config.common.yaml | 5 +- workflow/repo_data/config/config.default.yaml | 38 ++++++---- .../config/config.equivalence-usa.yaml | 8 +- .../repo_data/config/config.equivalence.yaml | 8 +- workflow/repo_data/config/config.sector.yaml | 4 +- workflow/rules/solve_electricity.smk | 8 +- workflow/rules/validate.smk | 6 +- workflow/scripts/opts/policy.py | 4 +- 24 files changed, 244 insertions(+), 189 deletions(-) diff --git a/.gitignore b/.gitignore index 30c107791..c93579e6a 100644 --- a/.gitignore +++ b/.gitignore @@ -11,8 +11,12 @@ dconf *.dot *.tar.gz -# configuration files -config/ +# per-user configuration files, seeded by init_pypsa_usa.sh. The canonical +# templates live in workflow/repo_data/config/ and ARE tracked — the old +# bare `config/` pattern matched that directory too and forced every template +# edit through `git add -f`. +/workflow/config/* +!/workflow/config/.gitkeep # generated files results/ @@ -266,7 +270,6 @@ TSWLatexianTemp* # Exceptions to gitignore connect.sh -/workflow/config/config.cluster.yaml /workflow/repo_data/dag.png !.pre-commit-config.yaml diff --git a/CLAUDE.md b/CLAUDE.md index ea45c3e6b..d77f149cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ PyPSA-USA is a Snakemake-orchestrated PyPSA workflow for capacity expansion, pro ## Running the workflow -**All `snakemake` invocations run from `workflow/`** — `cd workflow/` first. Snakemake auto-loads `config/config.cluster.yaml`, `config.common.yaml`, `config.plotting.yaml`, `config.api.yaml`, `config.sector.yaml` from `workflow/Snakefile`; the main configfile is passed via `--configfile`. +**All `snakemake` invocations run from `workflow/`** — `cd workflow/` first. `workflow/Snakefile` auto-loads the whole layered base out of the tracked templates: `repo_data/config/config.{cluster,common,plotting,api,sector,default}.yaml`, then the optional per-user overlays `config/config.api.yaml` and `config/config.cluster.yaml`, then whatever is passed via `--configfile`. Because `config.default.yaml` is a loaded layer, a scenario config is a sparse **overlay** — it only needs the keys it changes (nested mappings merge; lists and scalars are replaced wholesale). ```bash cd workflow @@ -20,7 +20,7 @@ Useful targets: - `rule data_model` — build everything up to the assembled-but-unsolved network (no solver). - `rule all` — full pipeline including solve and figures. - `--until ` to stop early, `-R ` to force re-execution. -- Tutorial config (`config/config.tutorial.yaml`, CA only, simpl=75, clusters=4m, 2050) is the smallest meaningful end-to-end run. +- Tutorial config (`repo_data/config/config.tutorial.yaml`, CA only, simpl=75, clusters=4m, 2050) is the smallest meaningful end-to-end run. HPC: edit `config/config.cluster.yaml` (account/partition/email) and `workflow/run_slurm.sh`, then `bash workflow/run_slurm.sh`. @@ -89,10 +89,11 @@ Defined in `workflow/Snakefile`: ## Configs -- `config/config.default.yaml` — primary user-facing config (Western, default knobs). -- `config/config.tutorial.yaml` — minimal CA-only smoke run. -- `workflow/repo_data/config/` mirrors `config/` and is the source for `docs/source/configtables/` documentation. -- Layered configs in `config/config.{cluster,common,api,plotting,sector}.yaml` are merged automatically by `Snakefile`; the main configfile only overrides what it needs to. +- `workflow/repo_data/config/` is canonical and is what the Snakefile loads. It is also the source for `docs/source/configtables/` documentation. +- `repo_data/config/config.default.yaml` — the scenario base layer: every user-facing knob, with defaults. Auto-loaded, and also the file users copy as a starting scenario. +- `repo_data/config/config.tutorial.yaml`, `config.test.yaml` — sparse overlays (only keys that differ from the base). `config.equivalence*.yaml` are deliberately self-contained because the Tier C harness replays them against a pinned upstream anchor that does not load the base. +- `workflow/config/` is untracked and holds only per-user files, seeded by `init_pypsa_usa.sh`: `config.default.yaml` (your scenario starting point), `config.api.yaml`, `config.cluster.yaml`. Do not add layered configs back into it. +- `policy_constraints/` CSVs are read straight from `repo_data/config/policy_constraints/`. ## Things to know before changing rules diff --git a/README.md b/README.md index 941ab539a..a3dc2786e 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ With [uv](https://docs.astral.sh/uv/) installed: ```bash git clone https://github.com/PyPSA/pypsa-usa.git cd pypsa-usa -bash init_pypsa_usa.sh # copy configuration templates into workflow/config +bash init_pypsa_usa.sh # seed the per-user config files into workflow/config cd workflow uv run snakemake -j1 --configfile config/config.default.yaml ``` diff --git a/docs/source/about-install.md b/docs/source/about-install.md index 4dbaf2d38..6cc9a2287 100644 --- a/docs/source/about-install.md +++ b/docs/source/about-install.md @@ -20,8 +20,14 @@ git clone git@github.com:PyPSA/pypsa-usa.git ## Step 2. Initialize Configuration files -From the command line, run the script `init_pypsa_usa.sh` to copy configuration file -templates into the `workflow/config` folder. +From the command line, run the script `init_pypsa_usa.sh` to seed the per-user +configuration files into the `workflow/config` folder. + +Only three files are copied — `config.default.yaml` (a starting point for your own +scenario config), `config.api.yaml` (API keys) and `config.cluster.yaml` (HPC account +settings). Every other configuration file is read by the workflow directly from the +tracked `workflow/repo_data/config/` templates, so it can never fall out of sync with +your checkout. The script is safe to re-run: existing files are left untouched. ```console bash init_pypsa_usa.sh @@ -103,4 +109,6 @@ Snakemake's internal job scheduler, not the optimization solver). The PyPSA-USA workflow leverages the EIA API in several steps. The default configuration activates dynamic fuel-cost prices, which requires EIA API key. You can quickly get your key by completing this [form](https://www.eia.gov/opendata/register.php). -The API key will be emailed to you, and you can copy the key into the `config.api.yaml` file. +The API key will be emailed to you. Paste it into `workflow/config/config.api.yaml`, or +export it as the `EIA_API_KEY` environment variable — the environment variable takes +precedence over the YAML value, which keeps the key out of your files entirely. diff --git a/docs/source/about-introduction.md b/docs/source/about-introduction.md index b388434bc..9437c042e 100644 --- a/docs/source/about-introduction.md +++ b/docs/source/about-introduction.md @@ -111,7 +111,7 @@ created under `workflow/` on first run. ```console ├── README.md ├── LICENSE.md -├── init_pypsa_usa.sh # one-time setup: copies default configs into place +├── init_pypsa_usa.sh # one-time setup: seeds the per-user config files ├── pyproject.toml # python dependencies (uv); pins pypsa/atlite/linopy ├── docs # this documentation (sphinx + myst) ├── tests # static + integration test suites @@ -119,8 +119,8 @@ created under `workflow/` on first run. ├── Snakefile # entry point: wildcards, paths, top-level rules ├── rules # snakemake rule definitions (*.smk) ├── scripts # python scripts executed by the rules - ├── config # your run configuration (config.default.yaml, ...) - ├── repo_data # small checked-in seed data (shapes, costs, dag) + ├── config # your per-user files (api keys, HPC, scenario configs) + ├── repo_data # checked-in seed data + the canonical config templates ├── envs # conda environment specification ├── data # downloaded raw data bundles ├── cutouts # atlite weather cutouts (optional, large) diff --git a/docs/source/about-usage.md b/docs/source/about-usage.md index decdcde3b..b52f00706 100644 --- a/docs/source/about-usage.md +++ b/docs/source/about-usage.md @@ -7,7 +7,13 @@ ## Set Configuration -To start, you'll want to set the proper network configuration for your studies purpose. The default configuration in `config/config.default.yaml` using the `western` interconnect and 33 nodes is a good place to start! +To start, you'll want to set the proper network configuration for your studies purpose. The +default configuration in `config/config.default.yaml` (seeded from the tracked template by +`init_pypsa_usa.sh`) using the `western` interconnect and 33 nodes is a good place to start! + +Your config file only needs to carry the keys you actually change: the workflow always +loads `repo_data/config/config.default.yaml` and the other layered files underneath it, and +your `--configfile` is merged on top. You can find more information on each configuration setting on the [configurations page](https://pypsa-usa.readthedocs.io/en/latest/config-configuration.html). @@ -45,7 +51,7 @@ snakemake data_model -j1 --configfile config/config.default.yaml ## Running on HPC Cluster -If you are running the workflow on an High-Performance Compute (HPC) cluster, you will first need to update the configuration settings in `config.cluster.yaml`. Update the account, partition, email, and chdir fields to match the information of your institutions cluster. +If you are running the workflow on an High-Performance Compute (HPC) cluster, you will first need to update the configuration settings in `workflow/config/config.cluster.yaml` (seeded by `init_pypsa_usa.sh`). Update the account, partition, email, and chdir fields to match the information of your institutions cluster. Next, identify the name of the configuration file you would like to run by editing the `run_slurm.sh` script. The default value is the `--configfile config/config.default.yaml`. diff --git a/docs/source/data-policies.md b/docs/source/data-policies.md index a78adeabd..e9c656449 100644 --- a/docs/source/data-policies.md +++ b/docs/source/data-policies.md @@ -15,9 +15,9 @@ PyPSA-USA integrates with the ReEDS capacity expansion model developed by NREL t ### Policy Data Default policy inputs are tracked in `workflow/repo_data/config/policy_constraints/` and are -copied into `workflow/config/policy_constraints/` — the untracked working config directory the -workflow reads — by `init_pypsa_usa.sh`. Each file can be used as shipped or replaced with -custom entries to explore new policy pathways. +read from there directly by the rules — there is no copy step. Each file can be used as +shipped, edited in place, or replaced by pointing the corresponding `electricity:` config +key at your own CSV to explore new policy pathways. - **Renewable Portfolio Standards (RPS)**: State RPS compliance trajectories derived from the NREL ReEDS model inputs (`reeds/rps_fraction.csv`), covering roughly 30 states with annual diff --git a/docs/source/release-notes.md b/docs/source/release-notes.md index 4e2715a0e..94d3a4b80 100644 --- a/docs/source/release-notes.md +++ b/docs/source/release-notes.md @@ -30,8 +30,10 @@ changes you will notice: `geospatial/`, `costs/`, `prices/`, `demand/`, `powerplants/`, ... — instead of per-interconnect flat folders. File names are unchanged. - Unused config keys and never-read rule parameters were removed. The canonical config - templates live in `workflow/repo_data/config/`; `workflow/config/` is no longer tracked in - git and is generated from those templates by `init_pypsa_usa.sh`. + templates live in `workflow/repo_data/config/` and the workflow loads its whole layered + base from there; `workflow/config/` is untracked and holds only the per-user files + (`config.api.yaml`, `config.cluster.yaml`, your own scenario configs) seeded by + `init_pypsa_usa.sh`. ### Correctness fixes validated by an equivalence harness diff --git a/init_pypsa_usa.sh b/init_pypsa_usa.sh index 0034744b8..a936e8626 100644 --- a/init_pypsa_usa.sh +++ b/init_pypsa_usa.sh @@ -1,13 +1,44 @@ #!/bin/bash +# One-time setup: seed the per-user configuration files. +# +# Everything else - config.common.yaml, config.plotting.yaml, +# config.sector.yaml, config.default.yaml, config.cluster.yaml and the +# policy_constraints/ CSVs - is loaded by workflow/Snakefile straight out of +# the tracked templates directory, so there is nothing to copy and nothing to +# keep in sync. Only the files below are genuinely user-owned. +# +# Safe to re-run: existing files are left untouched, missing ones are created. + +set -euo pipefail templates="workflow/repo_data/config" destination="workflow/config" -existing_files=$(ls "$destination" | grep -v ".gitkeep") - -if [ -z "$existing_files" ]; then - echo "Copying config files from '$templates' to '$destination'..." - cp -r "$templates"/* "$destination" -else - echo "Existing config files found in '$destination'. Delete the following files and rerun." - echo "$existing_files" -fi + +# Files copied into $destination for the user to edit. +# config.default.yaml starting point for your own scenario config +# config.api.yaml API keys (EIA); can be replaced by $EIA_API_KEY +# config.cluster.yaml HPC/SLURM account, partition, email +user_files=( + "config.default.yaml" + "config.api.yaml" + "config.cluster.yaml" +) + +mkdir -p "$destination" + +created=0 +for f in "${user_files[@]}"; do + if [ -e "$destination/$f" ]; then + echo "keeping existing $destination/$f" + else + cp "$templates/$f" "$destination/$f" + echo "created $destination/$f" + created=$((created + 1)) + fi +done + +echo +echo "Done ($created file(s) created)." +echo "Edit $destination/config.default.yaml (or copy it to" +echo "$destination/config..yaml) and run the workflow with:" +echo " cd workflow && snakemake -j1 --configfile config/config.default.yaml" diff --git a/tests/docs/test_docs_config.py b/tests/docs/test_docs_config.py index f6804ef12..236421e9e 100644 --- a/tests/docs/test_docs_config.py +++ b/tests/docs/test_docs_config.py @@ -11,9 +11,13 @@ ``yaml.safe_load`` to assert the top-level key set is exactly the expected one for each ``# docs : `` marker. -2. ``test_config_tree_sync`` — ``workflow/repo_data/config/`` is canonical and - ``init_pypsa_usa.sh`` copies it to ``workflow/config/``. The copies drift; - this test asserts they are byte-identical (excluding per-user files). +2. ``test_init_copies_only_user_owned_files`` / + ``test_snakefile_reads_layered_base_from_repo_data`` — + ``workflow/repo_data/config/`` is canonical AND is what the Snakefile + loads. ``init_pypsa_usa.sh`` seeds only the handful of genuinely per-user + files into ``workflow/config/``. These two tests keep that split intact: + copying a layered config back into ``config/``, or pointing a + ``configfile:`` at ``config/``, reintroduces the drift the layout removes. Dependency-light on purpose: stdlib + PyYAML only. """ @@ -211,36 +215,51 @@ def test_scoping_check_catches_old_costs_bug(): assert_slice_matches_marker("\n".join(sliced), "COSTS", "old-costs-bug-pattern") -def _init_script_dirs() -> tuple[Path, Path]: - """Read the template/destination dirs out of init_pypsa_usa.sh.""" +def _init_script() -> tuple[Path, Path, set[str]]: + """Read the template dir, destination dir and copied file list from init.""" script = (REPO_ROOT / "init_pypsa_usa.sh").read_text(encoding="utf-8") templates = re.search(r'templates="([^"]+)"', script) destination = re.search(r'destination="([^"]+)"', script) assert templates and destination, "could not parse init_pypsa_usa.sh" - return REPO_ROOT / templates.group(1), REPO_ROOT / destination.group(1) + block = re.search(r"user_files=\((.*?)\)", script, re.DOTALL) + assert block, "could not find the user_files array in init_pypsa_usa.sh" + names = set(re.findall(r'"([^"]+)"', block.group(1))) + assert names, "user_files array in init_pypsa_usa.sh is empty" + return REPO_ROOT / templates.group(1), REPO_ROOT / destination.group(1), names -# Per-user files: gitignored working copies that legitimately diverge from the -# repo_data templates (API keys, personal HPC account/walltime settings). -PER_USER_FILES = {"config.api.yaml", "config.cluster.yaml"} +def test_init_copies_only_user_owned_files(): + """init must seed exactly the per-user files, and each must exist as a template. - -def test_config_tree_sync(): - templates, destination = _init_script_dirs() + Everything else under ``workflow/repo_data/config/`` is loaded by + ``workflow/Snakefile`` straight from the tracked tree, so copying it into + ``workflow/config/`` would only reintroduce the drift this layout removes. + """ + templates, _destination, user_files = _init_script() assert templates.is_dir(), f"{templates} missing" - template_yamls = sorted(p for p in templates.rglob("*.yaml") if p.name not in PER_USER_FILES) - assert template_yamls, "no template yamls found" - - if not destination.is_dir() or not any(destination.rglob("*.yaml")): - pytest.skip("workflow/config is empty — init_pypsa_usa.sh has not been run") - - stale = [] - for template in template_yamls: - copy = destination / template.relative_to(templates) - if not copy.exists(): - stale.append(f"{copy.relative_to(REPO_ROOT)} missing") - elif copy.read_text(encoding="utf-8") != template.read_text(encoding="utf-8"): - stale.append(f"{copy.relative_to(REPO_ROOT)} differs from template") - - assert not stale, f"workflow/config has drifted from workflow/repo_data/config (re-copy the templates): {stale}" + missing = sorted(name for name in user_files if not (templates / name).exists()) + assert not missing, f"init_pypsa_usa.sh copies files with no template: {missing}" + + # A file init copies must NOT also be one the Snakefile reads from + # repo_data as an authoritative base layer that users cannot override — + # each of these is loaded as an overlay (api, cluster) or passed by hand + # with --configfile (default). + assert user_files <= { + "config.default.yaml", + "config.api.yaml", + "config.cluster.yaml", + "config.slurm.yaml", + }, f"unexpected per-user file in init_pypsa_usa.sh: {sorted(user_files)}" + + +def test_snakefile_reads_layered_base_from_repo_data(): + """The layered base must be read from the tracked templates, not from copies.""" + snakefile = (REPO_ROOT / "workflow" / "Snakefile").read_text(encoding="utf-8") + layered = re.findall(r'^configfile:\s*"([^"]+)"', snakefile, re.MULTILINE) + assert layered, "no unconditional configfile: directives found in workflow/Snakefile" + strays = [p for p in layered if not p.startswith("repo_data/config/")] + assert not strays, ( + "these layered configfiles are still read from the per-user config/ copy " + f"and will drift: {strays}" + ) diff --git a/tests/equivalence/build.py b/tests/equivalence/build.py index 41833cf7a..aa903aa03 100644 --- a/tests/equivalence/build.py +++ b/tests/equivalence/build.py @@ -40,7 +40,7 @@ from collections.abc import Iterable from pathlib import Path -from .paths import CONFIGFILE +from .paths import ANCHOR_CONFIGFILE, CONFIGFILE REPO = Path(__file__).resolve().parents[2] ANCHOR_SHA = "e7f8bd70" @@ -524,6 +524,16 @@ def apply_seam_adoption(wt: Path, applied_rules: list[str]) -> None: applied_rules.append("add_electricity") +def side_configfile(side: str) -> str: + """Where each side's copy of the shared harness config lives. + + The candidate loads it from the tracked ``repo_data/config/`` templates; + the anchor is a pinned upstream checkout that only looks in ``config/``, + and ``provision_anchor_worktree`` copies the file there. + """ + return ANCHOR_CONFIGFILE if side == "anchor" else CONFIGFILE + + def snakemake_cmd( target: str, jobs: int = 4, @@ -550,7 +560,7 @@ def build_side(side: str, target: str, jobs: int = 4, timeout: int = 10800) -> d assert side in ("candidate", "anchor") wt = provision_anchor_worktree() if side == "anchor" else REPO wf = wt / "workflow" - cmd = snakemake_cmd(target, jobs) + cmd = snakemake_cmd(target, jobs, side_configfile(side)) marker = wt / FORCE_RERUN_MARKER forced = marker.read_text().split() if marker.exists() else [] if forced: @@ -572,7 +582,7 @@ def build_side(side: str, target: str, jobs: int = 4, timeout: int = 10800) -> d def write_manifest(side: str, wt: Path, target: str, wall: float) -> dict: wf = wt / "workflow" sha = run(["git", "rev-parse", "HEAD"], cwd=wt).stdout.strip() - cfg = (wf / CONFIGFILE).read_bytes() + cfg = (wf / side_configfile(side)).read_bytes() manifest = { "side": side, "sha": sha, diff --git a/tests/equivalence/paths.py b/tests/equivalence/paths.py index 9a477a7fd..79bc3e582 100644 --- a/tests/equivalence/paths.py +++ b/tests/equivalence/paths.py @@ -22,9 +22,15 @@ RUN = "equivalence" INTERCONNECT = os.environ.get("EQ_INTERCONNECT", "western") UNTIL = os.environ.get("EQ_UNTIL", "") # 'assembled' = stop pairs at the assembled stage -CONFIGFILE = ( - "config/config.equivalence.yaml" if INTERCONNECT == "western" else f"config/config.equivalence-{INTERCONNECT}.yaml" +_CONFIG_NAME = ( + "config.equivalence.yaml" if INTERCONNECT == "western" else f"config.equivalence-{INTERCONNECT}.yaml" ) +# Candidate side reads the tracked template directly (its Snakefile no longer +# needs a config/ copy). The anchor is a pinned upstream checkout whose +# Snakefile still expects everything under config/, and build.py copies the +# shared harness config in there. +CONFIGFILE = f"repo_data/config/{_CONFIG_NAME}" +ANCHOR_CONFIGFILE = f"config/{_CONFIG_NAME}" CLUSTERS = "4" LL = "v1.0" OPTS = "REM-3h" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index a2500105a..74b6d7137 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -8,7 +8,6 @@ from __future__ import annotations import os -import shutil import subprocess from dataclasses import dataclass from pathlib import Path @@ -23,8 +22,6 @@ REPO_ROOT = Path(__file__).resolve().parents[2] WORKFLOW_DIR = REPO_ROOT / "workflow" -RUNTIME_CONFIG_DIR = WORKFLOW_DIR / "config" -TEMPLATE_CONFIG_DIR = WORKFLOW_DIR / "repo_data" / "config" DATA_DIRS = [WORKFLOW_DIR / "data", WORKFLOW_DIR / "cutouts", WORKFLOW_DIR / "repo_data"] @@ -33,10 +30,9 @@ def pytest_collection_modifyitems(config, items): The ``built`` session fixture runs snakemake once per session against the shared ``workflow/`` directory. With xdist, each worker is its own - session — they would race on ``.snakemake/`` locks, repeat the (slow) - snakemake build, and contend on the seeding step that copies templates - from ``repo_data/config/``. Until the fixture is refactored to use a - truly isolated per-worker workflow dir, refuse to run. + session — they would race on ``.snakemake/`` locks and repeat the (slow) + snakemake build. Until the fixture is refactored to use a truly isolated + per-worker workflow dir, refuse to run. """ if not any(item.get_closest_marker("integration") for item in items): return @@ -49,29 +45,6 @@ def pytest_collection_modifyitems(config, items): ) -@pytest.fixture(scope="session", autouse=True) -def _seed_runtime_configs(): - """Mirror ``init_pypsa_usa.sh``: copy any missing scenario configs + - ``policy_constraints/`` from ``workflow/repo_data/config/`` into - ``workflow/config/`` so snakemake has the inputs it expects. - Existing tracked files (e.g. ``config.common.yaml``) are left alone. - - Duplicated from ``tests/static/test_dag_dryrun.py`` so Tier B works - without depending on Tier A collection order. - """ - if not TEMPLATE_CONFIG_DIR.exists(): - return - RUNTIME_CONFIG_DIR.mkdir(parents=True, exist_ok=True) - for src in TEMPLATE_CONFIG_DIR.iterdir(): - dst = RUNTIME_CONFIG_DIR / src.name - if dst.exists(): - continue - if src.is_dir(): - shutil.copytree(src, dst) - else: - shutil.copy2(src, dst) - - @dataclass(frozen=True) class BuiltArtifacts: """Paths to the per-stage artifacts produced by the Tier B snakemake build. @@ -137,7 +110,7 @@ def built(tmp_path_factory) -> BuiltArtifacts: "--until", "cluster_network", "--configfile", - "config/config.test.yaml", + "repo_data/config/config.test.yaml", "--config", f"run={{name: '{run_name}', shared_cutouts: true}}", "-j", diff --git a/tests/static/test_config_keys.py b/tests/static/test_config_keys.py index dd4acfd5f..ae94ea03d 100644 --- a/tests/static/test_config_keys.py +++ b/tests/static/test_config_keys.py @@ -5,11 +5,10 @@ merged default config. Catches dead/typo'd config keys (the class of bug fixed in PR #10). -Canonical config templates live under ``workflow/repo_data/config/`` and -are copied into ``workflow/config/`` by ``init_pypsa_usa.sh``. The latter -also contains the committed ``config.common.yaml`` override. For maximum -coverage we merge the canonical templates and then overlay any tracked -runtime overrides on top. +The layered base lives under ``workflow/repo_data/config/`` and is loaded +from there by ``workflow/Snakefile``; ``workflow/config/`` only holds the +per-user overlays seeded by ``init_pypsa_usa.sh``. The merge order below +mirrors the ``configfile:`` chain in the Snakefile. """ from __future__ import annotations @@ -24,12 +23,10 @@ WORKFLOW_DIR = REPO_ROOT / "workflow" SCRIPT_DIR = WORKFLOW_DIR / "scripts" REPO_DATA_CONFIG_DIR = WORKFLOW_DIR / "repo_data" / "config" -RUNTIME_CONFIG_DIR = WORKFLOW_DIR / "config" -# Merge order matches the configfile chain in workflow/Snakefile plus -# config.default.yaml (the maximal scenario config). config.tutorial.yaml -# is intentionally omitted — it's a minimal opt-in subset used for fast -# DAG smoke tests, not a key-vocabulary source of truth. +# Merge order matches the configfile chain in workflow/Snakefile. +# config.tutorial.yaml is intentionally omitted — it's a sparse overlay used +# for fast DAG smoke tests, not a key-vocabulary source of truth. DEFAULT_CONFIGFILES = [ "config.cluster.yaml", "config.common.yaml", @@ -54,11 +51,7 @@ def _merge(a: dict, b: dict) -> dict: def _load_merged_config() -> dict: merged: dict = {} for name in DEFAULT_CONFIGFILES: - # Prefer the runtime workflow/config copy if present (it may carry - # tracked overrides), otherwise fall back to the canonical template. - runtime_path = RUNTIME_CONFIG_DIR / name - template_path = REPO_DATA_CONFIG_DIR / name - path = runtime_path if runtime_path.exists() else template_path + path = REPO_DATA_CONFIG_DIR / name if not path.exists(): continue with open(path) as f: @@ -145,6 +138,6 @@ def test_referenced_config_keys_exist(script, merged_config): unique_missing.append(keys) assert not unique_missing, ( f"{script.name}: snakemake.config keys not defined in any " - f"workflow/config/*.yaml or workflow/repo_data/config/*.yaml:\n" + f"workflow/repo_data/config/*.yaml:\n" + "\n".join(f" config[{']['.join(repr(k) for k in keys)}]" for keys in unique_missing) ) diff --git a/tests/static/test_dag_dryrun.py b/tests/static/test_dag_dryrun.py index 26310624f..84176912b 100644 --- a/tests/static/test_dag_dryrun.py +++ b/tests/static/test_dag_dryrun.py @@ -4,53 +4,29 @@ syntactically broken ``.smk`` files. Runs ``snakemake -n`` (dry-run only, no rule actually executes). -Snakemake reads scenario configs from ``workflow/config/``. Only -``config.common.yaml`` is tracked in git; the rest of the configs and -``policy_constraints/`` static inputs are seeded from -``workflow/repo_data/config/`` by ``init_pypsa_usa.sh`` at user-init -time. This test seeds any missing files in a session-scoped fixture so -it runs cleanly in CI without depending on the init script being run -ahead of pytest. +No seeding step: ``workflow/Snakefile`` reads its whole layered base and the +``policy_constraints/`` static inputs out of the tracked +``workflow/repo_data/config/`` tree, and treats the per-user files under +``workflow/config/`` as optional overlays. A fresh clone resolves without +``init_pypsa_usa.sh`` having been run. """ -import shutil import subprocess from pathlib import Path import pytest WORKFLOW_DIR = Path(__file__).resolve().parents[2] / "workflow" -RUNTIME_CONFIG_DIR = WORKFLOW_DIR / "config" -TEMPLATE_CONFIG_DIR = WORKFLOW_DIR / "repo_data" / "config" - - -@pytest.fixture(scope="session", autouse=True) -def _seed_runtime_configs(): - """Mirror ``init_pypsa_usa.sh``: copy any missing scenario configs + - ``policy_constraints/`` from ``workflow/repo_data/config/`` into - ``workflow/config/`` so ``snakemake -n`` has the inputs it expects. - Existing tracked files (e.g. ``config.common.yaml``) are left alone. - """ - if not TEMPLATE_CONFIG_DIR.exists(): - return - RUNTIME_CONFIG_DIR.mkdir(parents=True, exist_ok=True) - for src in TEMPLATE_CONFIG_DIR.iterdir(): - dst = RUNTIME_CONFIG_DIR / src.name - if dst.exists(): - continue - if src.is_dir(): - shutil.copytree(src, dst) - else: - shutil.copy2(src, dst) @pytest.mark.fast @pytest.mark.parametrize( "configfile,target", [ - ("config/config.tutorial.yaml", "cluster_network"), - ("config/config.tutorial.yaml", "solve_network"), - ("config/config.default.yaml", "cluster_network"), + ("repo_data/config/config.tutorial.yaml", "cluster_network"), + ("repo_data/config/config.tutorial.yaml", "solve_network"), + ("repo_data/config/config.default.yaml", "cluster_network"), + ("repo_data/config/config.test.yaml", "cluster_network"), ], ) def test_snakemake_dryrun_resolves(configfile, target): diff --git a/workflow/Snakefile b/workflow/Snakefile index b7b034041..8f329766d 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -7,6 +7,7 @@ from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider HTTP = HTTPRemoteProvider() +import os from os.path import normpath from itertools import chain from pathlib import Path @@ -93,18 +94,35 @@ wildcard_constraints: # config.sector.yaml sector-coupling defaults # config.default.yaml the canonical scenario base: every user-facing knob # -# config.default.yaml is loaded from the tracked repo_data/ templates rather -# than the per-user config/ copy: it is the workflow's base layer, not a -# user-owned file. Users copy it to config/config..yaml and pass -# that with --configfile, which then only needs to carry the keys it changes. -configfile: "config/config.cluster.yaml" -configfile: "config/config.common.yaml" -configfile: "config/config.plotting.yaml" -configfile: "config/config.api.yaml" -configfile: "config/config.sector.yaml" +# The whole base is read straight out of the tracked repo_data/config/ +# templates. It used to be read from the per-user config/ copies that +# init_pypsa_usa.sh seeds once and never refreshes, so the base silently +# drifted from the checkout. Only genuinely per-user files still live in +# config/ (see below); everything else is workflow-owned. +# +# Users copy config.default.yaml to config/config..yaml and pass it +# with --configfile; that file then only needs the keys it actually changes. +configfile: "repo_data/config/config.cluster.yaml" +configfile: "repo_data/config/config.common.yaml" +configfile: "repo_data/config/config.plotting.yaml" +configfile: "repo_data/config/config.api.yaml" +configfile: "repo_data/config/config.sector.yaml" configfile: "repo_data/config/config.default.yaml" +# User-owned overlays, seeded by init_pypsa_usa.sh. Optional so a fresh clone +# resolves without running init first; snakemake errors on a missing +# configfile, hence the guards. +if os.path.exists("config/config.api.yaml"): + + configfile: "config/config.api.yaml" + + +if os.path.exists("config/config.cluster.yaml"): + + configfile: "config/config.cluster.yaml" + + run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" diff --git a/workflow/repo_data/config/config.common.yaml b/workflow/repo_data/config/config.common.yaml index c0bfb917b..997a1db06 100644 --- a/workflow/repo_data/config/config.common.yaml +++ b/workflow/repo_data/config/config.common.yaml @@ -1,8 +1,9 @@ # ==================================================================== # PyPSA-USA — Common Configuration (always-loaded layered base) # ==================================================================== -# This file is auto-loaded by `workflow/Snakefile` underneath every -# main config. It holds settings that rarely change per scenario: +# This file is auto-loaded by `workflow/Snakefile` (straight from +# repo_data/, never a copy) underneath every main config. It holds +# settings that rarely change per scenario: # the renewable resource-quality dataset (atlite cutouts or GODEEEP # CFs + NREL land-access exclusions), atlite cutout geometries, # the line-type catalog, offshore-shape source, and the UCAP diff --git a/workflow/repo_data/config/config.default.yaml b/workflow/repo_data/config/config.default.yaml index 74a5d9840..09b51f54e 100644 --- a/workflow/repo_data/config/config.default.yaml +++ b/workflow/repo_data/config/config.default.yaml @@ -6,17 +6,25 @@ # capacity-expansion run at REeDS-zone resolution. # # Audience: users authoring a new scenario. Copy this file to -# `config/config..yaml`, edit only the keys you care -# about, and pass `--configfile config/config..yaml`. +# `config/config..yaml`, DELETE everything you do not +# change, and pass `--configfile config/config..yaml`. +# A scenario config is an overlay, not a fork: anything you leave out +# comes from this file. # -# How configs are merged: Snakemake auto-loads, in order, -# config/config.common.yaml (always-loaded layered base) -# config/config.api.yaml (API keys; gitignored) -# config/config.cluster.yaml (HPC scheduler settings) -# config/config.plotting.yaml (figure styling) -# config/config.sector.yaml (sector-coupling defaults) -# beneath whatever main config you pass via --configfile. Keys here -# override anything in those layered files. +# How configs are merged: workflow/Snakefile auto-loads, in order, +# repo_data/config/config.cluster.yaml (HPC scheduler settings) +# repo_data/config/config.common.yaml (renewable/atlite plumbing) +# repo_data/config/config.plotting.yaml (figure styling) +# repo_data/config/config.api.yaml (API-key placeholders) +# repo_data/config/config.sector.yaml (sector-coupling defaults) +# repo_data/config/config.default.yaml (THIS FILE — the scenario base) +# then the optional per-user overlays config/config.api.yaml and +# config/config.cluster.yaml, and finally whatever you pass with +# --configfile. Nested mappings merge; lists and scalars are REPLACED +# wholesale, so a list you change must be restated in full. +# +# Editing this file changes the base for every run in the checkout. +# Prefer putting scenario-specific values in your own config instead. # # Option reference: docs/source/config-configuration.md (Sphinx tables) # Wildcard reference: docs/source/config-wildcards.md @@ -144,7 +152,7 @@ electricity: # ---------------- Reserves & resource adequacy ---------------- SAFE_reservemargin: 0.14 # SAFE planning reserve margin (fraction of peak load); used with SAFE opt - SAFE_regional_reservemargins: 'config/policy_constraints/SAFE_regional_prm.csv' + SAFE_regional_reservemargins: 'repo_data/config/policy_constraints/SAFE_regional_prm.csv' operational_reserve: # GenX-style reserve requirement; only active when activate=true activate: false @@ -156,10 +164,10 @@ electricity: all: 0.15 # default ERM applied to every region (fraction of annual energy) # ---------------- Policy & emissions ---------------- - regional_Co2_limits: 'config/policy_constraints/regional_Co2_limits.csv' # per-region CO2 caps (tCO2/yr) - technology_capacity_targets: 'config/policy_constraints/technology_capacity_targets.csv' # forced min/max build by tech & region - portfolio_standards: 'config/policy_constraints/portfolio_standards.csv' # RPS/CES fractions by region - transmission_interface_limits: 'config/policy_constraints/transmission_interface_limits.csv' # MW caps on inter-region flows + regional_Co2_limits: 'repo_data/config/policy_constraints/regional_Co2_limits.csv' # per-region CO2 caps (tCO2/yr) + technology_capacity_targets: 'repo_data/config/policy_constraints/technology_capacity_targets.csv' # forced min/max build by tech & region + portfolio_standards: 'repo_data/config/policy_constraints/portfolio_standards.csv' # RPS/CES fractions by region + transmission_interface_limits: 'repo_data/config/policy_constraints/transmission_interface_limits.csv' # MW caps on inter-region flows # ---------------- Demand ---------------- demand: diff --git a/workflow/repo_data/config/config.equivalence-usa.yaml b/workflow/repo_data/config/config.equivalence-usa.yaml index 6d810db53..a84c459e5 100644 --- a/workflow/repo_data/config/config.equivalence-usa.yaml +++ b/workflow/repo_data/config/config.equivalence-usa.yaml @@ -91,10 +91,10 @@ electricity: # opts string; technology_capacity_targets (TCT opt) and # portfolio_standards (RPS opt) are the sibling keys read the same way. # Paths mirror config.default.yaml; the CSVs ship in - # config/policy_constraints/ in both checkouts. - regional_Co2_limits: 'config/policy_constraints/regional_Co2_limits.csv' - technology_capacity_targets: 'config/policy_constraints/technology_capacity_targets.csv' - portfolio_standards: 'config/policy_constraints/portfolio_standards.csv' + # repo_data/config/policy_constraints/ in both checkouts. + regional_Co2_limits: 'repo_data/config/policy_constraints/regional_Co2_limits.csv' + technology_capacity_targets: 'repo_data/config/policy_constraints/technology_capacity_targets.csv' + portfolio_standards: 'repo_data/config/policy_constraints/portfolio_standards.csv' conventional: unit_commitment: false diff --git a/workflow/repo_data/config/config.equivalence.yaml b/workflow/repo_data/config/config.equivalence.yaml index 0c95964d5..ff616916c 100644 --- a/workflow/repo_data/config/config.equivalence.yaml +++ b/workflow/repo_data/config/config.equivalence.yaml @@ -90,10 +90,10 @@ electricity: # opts string; technology_capacity_targets (TCT opt) and # portfolio_standards (RPS opt) are the sibling keys read the same way. # Paths mirror config.default.yaml; the CSVs ship in - # config/policy_constraints/ in both checkouts. - regional_Co2_limits: 'config/policy_constraints/regional_Co2_limits.csv' - technology_capacity_targets: 'config/policy_constraints/technology_capacity_targets.csv' - portfolio_standards: 'config/policy_constraints/portfolio_standards.csv' + # repo_data/config/policy_constraints/ in both checkouts. + regional_Co2_limits: 'repo_data/config/policy_constraints/regional_Co2_limits.csv' + technology_capacity_targets: 'repo_data/config/policy_constraints/technology_capacity_targets.csv' + portfolio_standards: 'repo_data/config/policy_constraints/portfolio_standards.csv' conventional: unit_commitment: false diff --git a/workflow/repo_data/config/config.sector.yaml b/workflow/repo_data/config/config.sector.yaml index 2e290fb65..39ba0458e 100644 --- a/workflow/repo_data/config/config.sector.yaml +++ b/workflow/repo_data/config/config.sector.yaml @@ -2,7 +2,7 @@ sector: # docs-co2 co2: sequestration_potential: 0 - policy: "config/policy_constraints/sector_co2_limits.csv" + policy: "repo_data/config/policy_constraints/sector_co2_limits.csv" # docs-ng natural_gas: imports: @@ -65,7 +65,7 @@ sector: transport_sector: brownfield: True # false to be implemented dynamic_costs: True # false to be implemented - ev_policy: "config/policy_constraints/ev_policy.csv" + ev_policy: "repo_data/config/policy_constraints/ev_policy.csv" must_run_evs: True modes: # false to be implemented vehicle: true diff --git a/workflow/rules/solve_electricity.smk b/workflow/rules/solve_electricity.smk index e9dacff97..0dbc73b6f 100644 --- a/workflow/rules/solve_electricity.smk +++ b/workflow/rules/solve_electricity.smk @@ -10,7 +10,7 @@ def pop_layout_input(wildcards): def ev_policy_input(wildcards): if wildcards["sector"] != "E": - return "config/policy_constraints/ev_policy.csv" + return "repo_data/config/policy_constraints/ev_policy.csv" else: return [] @@ -23,9 +23,9 @@ rule solve_network: network=NETWORKS + "{interconnect}/elec_s{simpl}_c{clusters}_ec_l{ll}_{opts}_{sector}.nc", flowgates="repo_data/ReEDS_Constraints/transmission/transmission_capacity_init_AC_ba_NARIS2024.csv", - safer_reeds="config/policy_constraints/reeds/prm_annual.csv", - rps_reeds="config/policy_constraints/reeds/rps_fraction.csv", - ces_reeds="config/policy_constraints/reeds/ces_fraction.csv", + safer_reeds="repo_data/config/policy_constraints/reeds/prm_annual.csv", + rps_reeds="repo_data/config/policy_constraints/reeds/rps_fraction.csv", + ces_reeds="repo_data/config/policy_constraints/reeds/ces_fraction.csv", pop_layout=pop_layout_input, ev_policy=ev_policy_input, output: diff --git a/workflow/rules/validate.smk b/workflow/rules/validate.smk index a6b968bbd..691e8ef48 100644 --- a/workflow/rules/validate.smk +++ b/workflow/rules/validate.smk @@ -6,9 +6,9 @@ rule solve_network_validation: network=NETWORKS + "{interconnect}/elec_s{simpl}_c{clusters}_ec_l{ll}_{opts}_{sector}.nc", flowgates="repo_data/ReEDS_Constraints/transmission/transmission_capacity_init_AC_ba_NARIS2024.csv", - safer_reeds="config/policy_constraints/reeds/prm_annual.csv", - rps_reeds="config/policy_constraints/reeds/rps_fraction.csv", - ces_reeds="config/policy_constraints/reeds/ces_fraction.csv", + safer_reeds="repo_data/config/policy_constraints/reeds/prm_annual.csv", + rps_reeds="repo_data/config/policy_constraints/reeds/rps_fraction.csv", + ces_reeds="repo_data/config/policy_constraints/reeds/ces_fraction.csv", output: network=RESULTS + "{interconnect}/networks/elec_s{simpl}_c{clusters}_ec_l{ll}_{opts}_{sector}_operations.nc", diff --git a/workflow/scripts/opts/policy.py b/workflow/scripts/opts/policy.py index b428dc73e..b7cb1f46a 100644 --- a/workflow/scripts/opts/policy.py +++ b/workflow/scripts/opts/policy.py @@ -33,7 +33,7 @@ def add_technology_capacity_target_constraints(n, config): Add minimum or maximum levels of generator nominal capacity per carrier for individual regions. Each constraint can be designated for a specified planning horizon in multi-period models. Opts and path for technology_capacity_targets.csv must be defined in config.yaml. - Default file is available at config/policy_constraints/technology_capacity_targets.csv. + Default file is available at repo_data/config/policy_constraints/technology_capacity_targets.csv. Parameters ---------- @@ -45,7 +45,7 @@ def add_technology_capacity_target_constraints(n, config): scenario: opts: [Co2L-TCT-24H] electricity: - technology_capacity_target: config/policy_constraints/technology_capacity_target.csv + technology_capacity_target: repo_data/config/policy_constraints/technology_capacity_target.csv """ tct_data = pd.read_csv(config["electricity"]["technology_capacity_targets"], comment="#") if tct_data.empty: From a6364628fe0b570d99d7a32558c0b015b6b1b592 Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 14:48:32 -0700 Subject: [PATCH 03/14] Give every top-level config key exactly one owning file config.common.yaml and config.default.yaml both defined renewable_weather_years, renewable_scenarios and co2: with identical values, and split renewable: and lines: across the two files - so answering "where is this configured?" meant reading both and knowing the load order. Each top-level key now lives in exactly one file, following where the bulk of each block already was: common owns renewable: (per-tech resource/land screens, plus `dataset`, which was the only thing default contributed) default owns lines: (the line-type catalog moves over to join s_max_pu / s_nom_max / max_extension / length_factor), plus co2:, renewable_weather_years and renewable_scenarios, whose duplicates in common are deleted. Both file headers now state the rule and the ownership split. Nothing moves out of the loaded set, so the merged config is byte-identical: the merge replay reports zero new differences against the origin/develop baseline, and a cross-layer scan finds no remaining duplicate top-level keys. No docs directive referenced the deleted # docs : RENEWABLE_DATASET or # docs : LINES_TYPES markers; the literalinclude slices that neighbour them still resolve to the same key sets (verified by tests/docs). Co-Authored-By: Claude Fable 5 --- workflow/repo_data/config/config.common.yaml | 64 ++++++------------- workflow/repo_data/config/config.default.yaml | 36 ++++++++--- 2 files changed, 45 insertions(+), 55 deletions(-) diff --git a/workflow/repo_data/config/config.common.yaml b/workflow/repo_data/config/config.common.yaml index 997a1db06..a972ad8bd 100644 --- a/workflow/repo_data/config/config.common.yaml +++ b/workflow/repo_data/config/config.common.yaml @@ -6,8 +6,20 @@ # settings that rarely change per scenario: # the renewable resource-quality dataset (atlite cutouts or GODEEEP # CFs + NREL land-access exclusions), atlite cutout geometries, -# the line-type catalog, offshore-shape source, and the UCAP -# forced-outage-rate table. +# offshore-shape source, and the UCAP forced-outage-rate table. +# +# ONE FILE PER TOP-LEVEL KEY. Every top-level key below is owned +# exclusively by this file; nothing here is also defined in +# config.default.yaml, so there is no "which layer wins?" question. +# The split follows where the bulk of each block already lived: +# renewable: HERE (per-technology resource + land screens, and +# `dataset`, which used to be duplicated in +# config.default.yaml with the same value) +# lines: config.default.yaml (the line-type catalog moved +# there to join s_max_pu and friends) +# renewable_weather_years / renewable_scenarios / co2: +# config.default.yaml (they were duplicated here with +# identical values) # # To override anything here, set the same key in your main config # file — keys in `--configfile
` take precedence over this one. @@ -28,17 +40,6 @@ pudl_path: s3://pudl.catalyst.coop/v2025.5.0 # versioned S3 prefix to PUDL parquet outputs (or a local file:// path) -# ==================================================================== -# GODEEEP renewable scenario selection -# ==================================================================== -# Consumed together with `renewable.dataset` below. `add_electricity.py:646` -# and `build_electricity.smk:166` index `renewable_scenarios[0]` directly -# whenever the dataset is godeeep, so both keys must be defined here. -# docs : RENEWABLE_SCENARIOS -renewable_weather_years: [2019] -renewable_scenarios: ["rcp85cooler"] # historical | rcp45hotter | rcp45cooler | rcp85hotter | rcp85cooler - - # ==================================================================== # RENEWABLE — per-technology resource and land-availability settings # ==================================================================== @@ -209,23 +210,6 @@ atlite: dy: 0.3 -# ==================================================================== -# LINES — pyPSA line-type catalog (mapped onto AC corridors by v_nom) -# ==================================================================== -# Each line in the base network is assigned the LineType matching its -# voltage. The string values reference pypsa.LineType library entries. -# docs : LINES_TYPES -lines: - types: # Temp values; replaced when actual conductor data is loaded - 115.: "Al/St 240/40 2-bundle 220.0" - 138.: "Al/St 240/40 2-bundle 220.0" - 161.: "Al/St 240/40 2-bundle 220.0" - 230.: "Al/St 240/40 2-bundle 220.0" - 345.: "Al/St 240/40 4-bundle 380.0" - 500.: "Al/St 560/50 4-bundle 750.0" - 765.: "Al/St 560/50 4-bundle 750.0" - - # ==================================================================== # OFFSHORE — offshore-shape source and OSW bus spacing # ==================================================================== @@ -238,21 +222,11 @@ offshore_network: bus_spacing: 25000 # meters between adjacent offshore buses (OSW network density) -# ==================================================================== -# CO2 — storage / pipeline-network defaults -# ==================================================================== -# Per-scenario configs override; keeping the keys defined here ensures every -# rule that reads ``config["co2"][...]`` resolves even when a config does not -# opt in. -# docs : -co2: - storage: false # [false, true] - network: - enable: false # [false, true] - capital_cost: 2736000 # USD per 12-inch-diameter mile - marginal_cost: 4 # USD/tCO2 - lifetime: 40 # years - discount_rate: 0.07 +# NB: `co2:` used to be duplicated here with the same values it has in +# config.default.yaml, which now owns the key outright. Its defaults are +# still always defined, because config.default.yaml is itself a loaded +# layer — so every rule reading config["co2"][...] still resolves. + # ==================================================================== # UCAP — Unforced Capacity for resource-adequacy accounting diff --git a/workflow/repo_data/config/config.default.yaml b/workflow/repo_data/config/config.default.yaml index 09b51f54e..88708266e 100644 --- a/workflow/repo_data/config/config.default.yaml +++ b/workflow/repo_data/config/config.default.yaml @@ -26,6 +26,12 @@ # Editing this file changes the base for every run in the checkout. # Prefer putting scenario-specific values in your own config instead. # +# ONE FILE PER TOP-LEVEL KEY. No top-level key here is also defined in +# config.common.yaml. `lines:`, `co2:`, `renewable_weather_years` and +# `renewable_scenarios` are owned here (the line-type catalog moved +# over from common); `renewable:` is owned by config.common.yaml +# (including `renewable.dataset`, which used to be duplicated here). +# # Option reference: docs/source/config-configuration.md (Sphinx tables) # Wildcard reference: docs/source/config-wildcards.md # Sector reference: docs/source/config-sectors.md @@ -43,15 +49,10 @@ run: shared_cutouts: true # true = share atlite cutouts across runs (recommended; cutouts are large and slow) validation: false # true = wire in back-casting validation plots (historical-only) -# ==================================================================== -# RENEWABLE DATASET — capacity-factor source for wind & solar -# ==================================================================== -# Sets which precomputed/atlite-derived CF time series the renewable -# attach pipeline consumes. `godeeep` requires `renewable_land_access` -# below; `atlite` requires a populated cutout (see config.common.yaml). -# docs : RENEWABLE_DATASET -renewable: - dataset: godeeep # atlite | godeeep — see docs/source/config-configuration.md#renewable-godeeep +# NB: the whole `renewable:` block — including `renewable.dataset` +# (atlite | godeeep), which used to be duplicated here — lives in +# config.common.yaml, which owns that key outright. See +# docs/source/config-configuration.md#renewable-godeeep. # ==================================================================== @@ -116,7 +117,9 @@ snapshots: inclusive: "left" # pandas.date_range inclusive arg; left | right | both | neither # GODEEEP future-scenario controls. Used only when renewable.dataset = godeeep -# and renewable_scenarios != "historical". +# (set in config.common.yaml) and renewable_scenarios != "historical". +# `add_electricity.py` and `build_electricity.smk` index renewable_scenarios[0] +# directly whenever the dataset is godeeep, so the key must stay defined. # docs : RENEWABLE_SCENARIOS renewable_scenarios: ["rcp85cooler"] # climate scenario; historical | rcp45cooler | rcp45hotter | rcp85cooler | rcp85hotter # renewable_scenario_years: [2030] # OPTIONAL: explicit list of GODEEEP scenario years; defaults to planning_horizons @@ -227,6 +230,19 @@ lines: max_extension: 20000 # MW cap on total network-wide line expansion length_factor: 1.25 # multiplier on geographic length to account for routing/right-of-way + # pyPSA line-type catalog, mapped onto AC corridors by v_nom. Every line in + # the base network is assigned the LineType matching its voltage; values + # reference pypsa.LineType library entries. (Moved here from + # config.common.yaml so `lines:` is defined in exactly one file.) + types: # Temp values; replaced when actual conductor data is loaded + 115.: "Al/St 240/40 2-bundle 220.0" + 138.: "Al/St 240/40 2-bundle 220.0" + 161.: "Al/St 240/40 2-bundle 220.0" + 230.: "Al/St 240/40 2-bundle 220.0" + 345.: "Al/St 240/40 4-bundle 380.0" + 500.: "Al/St 560/50 4-bundle 750.0" + 765.: "Al/St 560/50 4-bundle 750.0" + # ==================================================================== # LINKS — HVDC and controllable links (DC ties, internal HVDC) From ca0fa7be729f777b39fa49560f6059cc130a1241 Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 14:50:53 -0700 Subject: [PATCH 04/14] Rename config.cluster.yaml to config.slurm.yaml and unify per-rule walltimes The file has always been about the job scheduler, but its name collided with the `{clusters}` wildcard and the `clustering:` config section, so "the cluster config" was ambiguous every time it came up. Per-rule wall times were specified in three places. config.default.yaml's top-level `walltime:` block is the one the rules actually read (`config_provider("walltime", "")` -> `resources: walltime=` -> `sbatch --time {resources.walltime}`). config.cluster.yaml ALSO carried per-rule `{rule}: {walltime: ...}` blocks with different values, which nothing read: run_slurm.sh only dereferences `{cluster.account/email/ partition/output/error}` from `__default__`, and takes the time limit from resources. Those dead blocks were still injected into the merged config as top-level keys named after rules. config.slurm.yaml now holds `__default__` (for --cluster-config) plus the one `walltime:` block, and config.default.yaml's copy is deleted with a comment distinguishing it from `solving.walltime`, which caps the solver rather than the scheduler job. Live values are preserved exactly (config.default.yaml's won); the two rules that only existed in the dead blocks, aggregate_egs and solve_network_validation, are folded in - solve_network_validation is read with no fallback and had been resolving to None. Merged-config diff: the nine dead `.walltime` top-level keys are gone, `walltime.aggregate_egs` and `walltime.solve_network_validation` are added, and every previously-live walltime value is unchanged. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 6 +- docs/source/about-install.md | 2 +- docs/source/about-usage.md | 2 +- docs/source/config-configuration.md | 13 ++-- docs/source/release-notes.md | 2 +- init_pypsa_usa.sh | 6 +- tests/docs/test_docs_config.py | 1 - tests/equivalence/build.py | 3 + tests/static/test_config_keys.py | 2 +- workflow/Snakefile | 8 +-- workflow/repo_data/config/config.cluster.yaml | 36 ----------- workflow/repo_data/config/config.default.yaml | 24 ++------ workflow/repo_data/config/config.slurm.yaml | 61 +++++++++++++++++++ workflow/repo_data/config/config.test.yaml | 2 +- .../repo_data/config/config.tutorial.yaml | 2 +- workflow/run_slurm.sh | 4 +- 16 files changed, 96 insertions(+), 78 deletions(-) delete mode 100644 workflow/repo_data/config/config.cluster.yaml create mode 100644 workflow/repo_data/config/config.slurm.yaml diff --git a/CLAUDE.md b/CLAUDE.md index d77f149cf..18e4964fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,7 @@ PyPSA-USA is a Snakemake-orchestrated PyPSA workflow for capacity expansion, pro ## Running the workflow -**All `snakemake` invocations run from `workflow/`** — `cd workflow/` first. `workflow/Snakefile` auto-loads the whole layered base out of the tracked templates: `repo_data/config/config.{cluster,common,plotting,api,sector,default}.yaml`, then the optional per-user overlays `config/config.api.yaml` and `config/config.cluster.yaml`, then whatever is passed via `--configfile`. Because `config.default.yaml` is a loaded layer, a scenario config is a sparse **overlay** — it only needs the keys it changes (nested mappings merge; lists and scalars are replaced wholesale). +**All `snakemake` invocations run from `workflow/`** — `cd workflow/` first. `workflow/Snakefile` auto-loads the whole layered base out of the tracked templates: `repo_data/config/config.{slurm,common,plotting,api,sector,default}.yaml`, then the optional per-user overlays `config/config.api.yaml` and `config/config.slurm.yaml`, then whatever is passed via `--configfile`. Because `config.default.yaml` is a loaded layer, a scenario config is a sparse **overlay** — it only needs the keys it changes (nested mappings merge; lists and scalars are replaced wholesale). ```bash cd workflow @@ -22,7 +22,7 @@ Useful targets: - `--until ` to stop early, `-R ` to force re-execution. - Tutorial config (`repo_data/config/config.tutorial.yaml`, CA only, simpl=75, clusters=4m, 2050) is the smallest meaningful end-to-end run. -HPC: edit `config/config.cluster.yaml` (account/partition/email) and `workflow/run_slurm.sh`, then `bash workflow/run_slurm.sh`. +HPC: edit `config/config.slurm.yaml` (account/partition/email; it also holds the single per-rule `walltime:` block) and `workflow/run_slurm.sh`, then `bash workflow/run_slurm.sh`. ## Tests and lint @@ -92,7 +92,7 @@ Defined in `workflow/Snakefile`: - `workflow/repo_data/config/` is canonical and is what the Snakefile loads. It is also the source for `docs/source/configtables/` documentation. - `repo_data/config/config.default.yaml` — the scenario base layer: every user-facing knob, with defaults. Auto-loaded, and also the file users copy as a starting scenario. - `repo_data/config/config.tutorial.yaml`, `config.test.yaml` — sparse overlays (only keys that differ from the base). `config.equivalence*.yaml` are deliberately self-contained because the Tier C harness replays them against a pinned upstream anchor that does not load the base. -- `workflow/config/` is untracked and holds only per-user files, seeded by `init_pypsa_usa.sh`: `config.default.yaml` (your scenario starting point), `config.api.yaml`, `config.cluster.yaml`. Do not add layered configs back into it. +- `workflow/config/` is untracked and holds only per-user files, seeded by `init_pypsa_usa.sh`: `config.default.yaml` (your scenario starting point), `config.api.yaml`, `config.slurm.yaml`. Do not add layered configs back into it. - `policy_constraints/` CSVs are read straight from `repo_data/config/policy_constraints/`. ## Things to know before changing rules diff --git a/docs/source/about-install.md b/docs/source/about-install.md index 6cc9a2287..296b97a1d 100644 --- a/docs/source/about-install.md +++ b/docs/source/about-install.md @@ -24,7 +24,7 @@ From the command line, run the script `init_pypsa_usa.sh` to seed the per-user configuration files into the `workflow/config` folder. Only three files are copied — `config.default.yaml` (a starting point for your own -scenario config), `config.api.yaml` (API keys) and `config.cluster.yaml` (HPC account +scenario config), `config.api.yaml` (API keys) and `config.slurm.yaml` (HPC account settings). Every other configuration file is read by the workflow directly from the tracked `workflow/repo_data/config/` templates, so it can never fall out of sync with your checkout. The script is safe to re-run: existing files are left untouched. diff --git a/docs/source/about-usage.md b/docs/source/about-usage.md index b52f00706..50c9aa4b9 100644 --- a/docs/source/about-usage.md +++ b/docs/source/about-usage.md @@ -51,7 +51,7 @@ snakemake data_model -j1 --configfile config/config.default.yaml ## Running on HPC Cluster -If you are running the workflow on an High-Performance Compute (HPC) cluster, you will first need to update the configuration settings in `workflow/config/config.cluster.yaml` (seeded by `init_pypsa_usa.sh`). Update the account, partition, email, and chdir fields to match the information of your institutions cluster. +If you are running the workflow on an High-Performance Compute (HPC) cluster, you will first need to update the configuration settings in `workflow/config/config.slurm.yaml` (seeded by `init_pypsa_usa.sh`). Update the account, partition, email, and chdir fields to match the information of your institutions cluster. Next, identify the name of the configuration file you would like to run by editing the `run_slurm.sh` script. The default value is the `--configfile config/config.default.yaml`. diff --git a/docs/source/config-configuration.md b/docs/source/config-configuration.md index f6f6b38d6..b9cb6acfa 100644 --- a/docs/source/config-configuration.md +++ b/docs/source/config-configuration.md @@ -615,13 +615,16 @@ networks. (walltime_cf)= ## `walltime` -Per-rule wall-time overrides consumed as Snakemake `walltime` resources, used by the SLURM -profile when submitting jobs to an HPC scheduler (see `config.cluster.yaml` and -`workflow/run_slurm.sh`). Rules not listed here fall back to per-rule defaults defined in the -workflow. Local runs ignore these values. +Per-rule wall-time overrides consumed as Snakemake `walltime` resources and forwarded to +`sbatch --time` by `workflow/run_slurm.sh`. This is the single source for per-rule wall times; +it lives in `config.slurm.yaml` (renamed from `config.cluster.yaml`, whose name collided with +the `{clusters}` wildcard and the `clustering:` section). Rules not listed here fall back to +per-rule defaults defined in the workflow. Local runs ignore these values. + +Not to be confused with `solving: walltime:`, which caps the *solver*, not the scheduler job. ```{eval-rst} -.. literalinclude:: ../../workflow/repo_data/config/config.default.yaml +.. literalinclude:: ../../workflow/repo_data/config/config.slurm.yaml :language: yaml :start-after: # docs : WALLTIME :end-before: # docs : diff --git a/docs/source/release-notes.md b/docs/source/release-notes.md index 94d3a4b80..6dd5fec63 100644 --- a/docs/source/release-notes.md +++ b/docs/source/release-notes.md @@ -32,7 +32,7 @@ changes you will notice: - Unused config keys and never-read rule parameters were removed. The canonical config templates live in `workflow/repo_data/config/` and the workflow loads its whole layered base from there; `workflow/config/` is untracked and holds only the per-user files - (`config.api.yaml`, `config.cluster.yaml`, your own scenario configs) seeded by + (`config.api.yaml`, `config.slurm.yaml`, your own scenario configs) seeded by `init_pypsa_usa.sh`. ### Correctness fixes validated by an equivalence harness diff --git a/init_pypsa_usa.sh b/init_pypsa_usa.sh index a936e8626..5004f55c9 100644 --- a/init_pypsa_usa.sh +++ b/init_pypsa_usa.sh @@ -2,7 +2,7 @@ # One-time setup: seed the per-user configuration files. # # Everything else - config.common.yaml, config.plotting.yaml, -# config.sector.yaml, config.default.yaml, config.cluster.yaml and the +# config.sector.yaml, config.default.yaml and the # policy_constraints/ CSVs - is loaded by workflow/Snakefile straight out of # the tracked templates directory, so there is nothing to copy and nothing to # keep in sync. Only the files below are genuinely user-owned. @@ -17,11 +17,11 @@ destination="workflow/config" # Files copied into $destination for the user to edit. # config.default.yaml starting point for your own scenario config # config.api.yaml API keys (EIA); can be replaced by $EIA_API_KEY -# config.cluster.yaml HPC/SLURM account, partition, email +# config.slurm.yaml HPC/SLURM account, partition, email user_files=( "config.default.yaml" "config.api.yaml" - "config.cluster.yaml" + "config.slurm.yaml" ) mkdir -p "$destination" diff --git a/tests/docs/test_docs_config.py b/tests/docs/test_docs_config.py index 236421e9e..412e77101 100644 --- a/tests/docs/test_docs_config.py +++ b/tests/docs/test_docs_config.py @@ -248,7 +248,6 @@ def test_init_copies_only_user_owned_files(): assert user_files <= { "config.default.yaml", "config.api.yaml", - "config.cluster.yaml", "config.slurm.yaml", }, f"unexpected per-user file in init_pypsa_usa.sh: {sorted(user_files)}" diff --git a/tests/equivalence/build.py b/tests/equivalence/build.py index aa903aa03..bd2d049a8 100644 --- a/tests/equivalence/build.py +++ b/tests/equivalence/build.py @@ -58,6 +58,9 @@ ADD_ELECTRICITY_SCRIPT = "workflow/scripts/add_electricity.py" FORCE_RERUN_MARKER = ".eq-force-rerun" # rules to -R once after a newly applied patch +# The ANCHOR's layered set (pinned upstream e7f8bd70), not the candidate's — +# it still reads every layer out of config/ and still calls the SLURM file +# config.cluster.yaml. LAYERED_CONFIGS = [ "config.cluster.yaml", "config.plotting.yaml", diff --git a/tests/static/test_config_keys.py b/tests/static/test_config_keys.py index ae94ea03d..c1384bb62 100644 --- a/tests/static/test_config_keys.py +++ b/tests/static/test_config_keys.py @@ -28,7 +28,7 @@ # config.tutorial.yaml is intentionally omitted — it's a sparse overlay used # for fast DAG smoke tests, not a key-vocabulary source of truth. DEFAULT_CONFIGFILES = [ - "config.cluster.yaml", + "config.slurm.yaml", "config.common.yaml", "config.plotting.yaml", "config.api.yaml", diff --git a/workflow/Snakefile b/workflow/Snakefile index 8f329766d..da15c1af3 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -87,7 +87,7 @@ wildcard_constraints: # REPLACED wholesale), and anything the user passes via `--configfile` is # merged last, on top of everything below. # -# config.cluster.yaml HPC scheduler settings +# config.slurm.yaml HPC scheduler settings + per-rule wall times # config.common.yaml renewable/atlite plumbing that rarely changes # config.plotting.yaml figure styling # config.api.yaml API keys (gitignored) @@ -102,7 +102,7 @@ wildcard_constraints: # # Users copy config.default.yaml to config/config..yaml and pass it # with --configfile; that file then only needs the keys it actually changes. -configfile: "repo_data/config/config.cluster.yaml" +configfile: "repo_data/config/config.slurm.yaml" configfile: "repo_data/config/config.common.yaml" configfile: "repo_data/config/config.plotting.yaml" configfile: "repo_data/config/config.api.yaml" @@ -118,9 +118,9 @@ if os.path.exists("config/config.api.yaml"): configfile: "config/config.api.yaml" -if os.path.exists("config/config.cluster.yaml"): +if os.path.exists("config/config.slurm.yaml"): - configfile: "config/config.cluster.yaml" + configfile: "config/config.slurm.yaml" run = config.get("run", {}) diff --git a/workflow/repo_data/config/config.cluster.yaml b/workflow/repo_data/config/config.cluster.yaml deleted file mode 100644 index abb83f27e..000000000 --- a/workflow/repo_data/config/config.cluster.yaml +++ /dev/null @@ -1,36 +0,0 @@ -__default__: - account: - partition: - email: - walltime: 00:30:00 # time limit for each job - cpus_per_task: 1 # number of cores per job - chdir: - output: logs/{rule}/log-%j.out - error: logs/{rule}/errlog-%j.err - -build_renewable_profiles: - walltime: 06:00:00 - -add_electricity: - walltime: 06:00:00 - -aggregate_to_substations: - walltime: 09:00:00 - -cluster_resources: - walltime: 09:00:00 - -aggregate_egs: - walltime: 01:00:00 - -cluster_network: - walltime: 09:00:00 - -solve_network: - walltime: '24:00:00' - -solve_network_validation: - walltime: 09:00:00 - -add_demand: - walltime: 02:00:00 diff --git a/workflow/repo_data/config/config.default.yaml b/workflow/repo_data/config/config.default.yaml index 88708266e..edb64a278 100644 --- a/workflow/repo_data/config/config.default.yaml +++ b/workflow/repo_data/config/config.default.yaml @@ -12,14 +12,14 @@ # comes from this file. # # How configs are merged: workflow/Snakefile auto-loads, in order, -# repo_data/config/config.cluster.yaml (HPC scheduler settings) +# repo_data/config/config.slurm.yaml (HPC scheduler + per-rule walltime) # repo_data/config/config.common.yaml (renewable/atlite plumbing) # repo_data/config/config.plotting.yaml (figure styling) # repo_data/config/config.api.yaml (API-key placeholders) # repo_data/config/config.sector.yaml (sector-coupling defaults) # repo_data/config/config.default.yaml (THIS FILE — the scenario base) # then the optional per-user overlays config/config.api.yaml and -# config/config.cluster.yaml, and finally whatever you pass with +# config/config.slurm.yaml, and finally whatever you pass with # --configfile. Nested mappings merge; lists and scalars are REPLACED # wholesale, so a list you change must be restated in full. # @@ -461,22 +461,10 @@ solving: cbc-default: {} # used in CI glpk-default: {} # used in CI - walltime: "12:00:00" # solver wall time cap (HH:MM:SS); enforced via solver hooks - - -# ==================================================================== -# WALLTIME — per-rule HPC walltime overrides (only used by SLURM driver) -# ==================================================================== -# docs : WALLTIME -walltime: - build_renewable_profiles: '04:00:00' - build_fuel_prices: '00:20:00' - add_demand: '02:00:00' - add_electricity: '04:00:00' - aggregate_to_substations: '02:00:00' - cluster_resources: '05:00:00' - cluster_network: '04:00:00' - solve_network: '20:00:00' + # Solver-side wall-time cap (HH:MM:SS), enforced via solver hooks. This is + # NOT the scheduler's job time limit: per-rule HPC wall times live in the + # single `walltime:` block in config.slurm.yaml. + walltime: "12:00:00" # ==================================================================== diff --git a/workflow/repo_data/config/config.slurm.yaml b/workflow/repo_data/config/config.slurm.yaml new file mode 100644 index 000000000..7335df81f --- /dev/null +++ b/workflow/repo_data/config/config.slurm.yaml @@ -0,0 +1,61 @@ +# ==================================================================== +# PyPSA-USA — SLURM / HPC execution settings +# ==================================================================== +# Renamed from config.cluster.yaml, whose name collided with the +# `{clusters}` wildcard and the `clustering:` config section while having +# nothing to do with either — it has always been about the job scheduler. +# +# This file is used in two ways: +# +# 1. `--cluster-config config/config.slurm.yaml` (see workflow/run_slurm.sh) +# resolves `{cluster.account}`, `{cluster.partition}`, `{cluster.email}`, +# `{cluster.output}` and `{cluster.error}` from `__default__` below. +# Fill those in with your institution's values in YOUR copy under +# workflow/config/ (seeded by init_pypsa_usa.sh); the tracked template +# here stays blank. +# +# 2. It is a `configfile:` layer, which is what puts `walltime:` into the +# merged config. That block is THE single source of per-rule wall times: +# rules read it as `config_provider("walltime", "")` and pass the +# result to `resources: walltime=`, which run_slurm.sh forwards to sbatch +# as `--time {resources.walltime}`. It used to be duplicated - partly in +# config.default.yaml under `walltime:` (the values the rules actually +# read) and partly here as per-rule `{rule}: {walltime: ...}` blocks that +# nothing read, because run_slurm.sh takes the time from resources rather +# than from `{cluster.walltime}`. +# +# `solving.walltime` and `solving.mem` in config.default.yaml are a different +# concern: they are the solver's own limits, not the scheduler's. +# ==================================================================== + +# sbatch defaults consumed via {cluster.*} under --cluster-config. +__default__: + account: + partition: + email: + walltime: 00:30:00 # fallback job time limit for {cluster.walltime} + cpus_per_task: 1 # number of cores per job + chdir: + output: logs/{rule}/log-%j.out + error: logs/{rule}/errlog-%j.err + + +# ==================================================================== +# WALLTIME — per-rule wall times requested from the scheduler +# ==================================================================== +# Rules not listed here fall back to the per-rule defaults declared inline in +# workflow/rules/*.smk. Local (non-HPC) runs ignore these values entirely. +# docs : WALLTIME +walltime: + build_renewable_profiles: '04:00:00' + build_fuel_prices: '00:20:00' + add_demand: '02:00:00' + add_electricity: '04:00:00' + aggregate_to_substations: '02:00:00' + aggregate_egs: '01:00:00' + cluster_resources: '05:00:00' + cluster_network: '04:00:00' + solve_network: '20:00:00' + solve_network_validation: '09:00:00' + +# docs : end diff --git a/workflow/repo_data/config/config.test.yaml b/workflow/repo_data/config/config.test.yaml index 986aff79f..bf90c8a84 100644 --- a/workflow/repo_data/config/config.test.yaml +++ b/workflow/repo_data/config/config.test.yaml @@ -5,7 +5,7 @@ # <5 minutes. DO NOT use as a user-facing entry point. # # This is a SPARSE OVERLAY. `workflow/Snakefile` already loads the -# layered base (config.cluster / config.common / config.plotting / +# layered base (config.slurm / config.common / config.plotting / # config.api / config.sector / config.default) underneath it, so only # keys that DIFFER from that merged base belong here. # diff --git a/workflow/repo_data/config/config.tutorial.yaml b/workflow/repo_data/config/config.tutorial.yaml index 54a998939..5dd852fb8 100644 --- a/workflow/repo_data/config/config.tutorial.yaml +++ b/workflow/repo_data/config/config.tutorial.yaml @@ -10,7 +10,7 @@ # uv run snakemake -j1 --configfile repo_data/config/config.tutorial.yaml # # This is a SPARSE OVERLAY. `workflow/Snakefile` already loads the -# layered base (config.cluster / config.common / config.plotting / +# layered base (config.slurm / config.common / config.plotting / # config.api / config.sector / config.default) underneath it, so only # keys that DIFFER from that merged base belong here. Everything not # listed — solver options, costs, clustering strategies, lines/links, diff --git a/workflow/run_slurm.sh b/workflow/run_slurm.sh index ce020b19e..56bdd4776 100644 --- a/workflow/run_slurm.sh +++ b/workflow/run_slurm.sh @@ -1,3 +1,3 @@ -# SLURM specifications made in default.cluster.yaml & the individual rules +# SLURM specifications live in config/config.slurm.yaml & the individual rules # GRB_LICENSE_FILE=/share/software/user/restricted/gurobi/11.0.2/licenses/gurobi.lic⁠ -snakemake --cluster "sbatch -A {cluster.account} --mail-type ALL --mail-user {cluster.email} -p {cluster.partition} -o {cluster.output} -e {cluster.error} -c {threads} --mem {resources.mem_mb} --time {resources.walltime}" --cluster-config config/config.cluster.yaml --jobs 20 --latency-wait 60 --rerun-incomplete --configfile config/CH1/config.tamu.single_horizon.bau.yaml +snakemake --cluster "sbatch -A {cluster.account} --mail-type ALL --mail-user {cluster.email} -p {cluster.partition} -o {cluster.output} -e {cluster.error} -c {threads} --mem {resources.mem_mb} --time {resources.walltime}" --cluster-config config/config.slurm.yaml --jobs 20 --latency-wait 60 --rerun-incomplete --configfile config/CH1/config.tamu.single_horizon.bau.yaml From e736ea55e247372c7facda9c736955600341bd0d Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 14:52:46 -0700 Subject: [PATCH 05/14] Resolve the EIA API key from $EIA_API_KEY, falling back to yaml The key was reachable only by pasting it into a yaml file inside the checkout. It is now read once in workflow/Snakefile, where the environment wins over config/config.api.yaml, so the secret can stay out of the repo entirely. A dozen rules dereference config["api"]["eia"] at parse time, so the override has to land before the include: statements - it does. The tracked repo_data/config/config.api.yaml keeps the key defined (as null) and documents both routes; the per-user copy under config/ is now an optional overlay. The two __main__ debug blocks in eia.py and build_natural_gas.py follow the same precedence, and no longer hard-require a config/ copy that a fresh clone does not have. Verified: with EIA_API_KEY set, the merged config carries the environment value; unset, it carries the yaml value (null on a bare checkout). Co-Authored-By: Claude Fable 5 --- workflow/Snakefile | 8 ++++++++ workflow/repo_data/config/config.api.yaml | 19 +++++++++++++++++-- workflow/scripts/build_natural_gas.py | 10 +++++++--- workflow/scripts/eia.py | 10 +++++++--- 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index da15c1af3..be971dc97 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -123,6 +123,14 @@ if os.path.exists("config/config.slurm.yaml"): configfile: "config/config.slurm.yaml" +# EIA API key. The environment wins over the YAML so the key can stay out of +# the checkout entirely; config/config.api.yaml remains the fallback. Resolved +# here, once, because a dozen rules dereference config["api"]["eia"] directly +# at parse time (below, via the include: statements). +config.setdefault("api", {}) +config["api"]["eia"] = os.environ.get("EIA_API_KEY") or config["api"].get("eia") + + run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" diff --git a/workflow/repo_data/config/config.api.yaml b/workflow/repo_data/config/config.api.yaml index 65e8336d2..6f835af8e 100644 --- a/workflow/repo_data/config/config.api.yaml +++ b/workflow/repo_data/config/config.api.yaml @@ -1,4 +1,19 @@ -# holds user api keys +# ==================================================================== +# PyPSA-USA — API keys +# ==================================================================== +# This tracked template only defines the key with a null value, so the +# workflow always resolves `config["api"]["eia"]`. Supply the real key +# either way: +# +# 1. Environment (preferred — keeps the secret out of the checkout): +# export EIA_API_KEY= +# workflow/Snakefile reads $EIA_API_KEY and it WINS over the yaml. +# +# 2. YAML: run `bash init_pypsa_usa.sh`, then paste the key into your +# own gitignored copy at workflow/config/config.api.yaml. +# +# Register for a free key at https://www.eia.gov/opendata/register.php +# ==================================================================== api: - eia: + eia: diff --git a/workflow/scripts/build_natural_gas.py b/workflow/scripts/build_natural_gas.py index 040fb17e2..45d4a0dd5 100644 --- a/workflow/scripts/build_natural_gas.py +++ b/workflow/scripts/build_natural_gas.py @@ -14,6 +14,7 @@ import logging import math +import os from abc import ABC, abstractmethod from math import pi from typing import Any @@ -1319,9 +1320,12 @@ def build_natural_gas( "../resources/Washington/networks/western/elec_s10_c4m_ec_lv1.0_3h.nc", ) year = 2018 - with open("./../config/config.api.yaml") as file: - yaml_data = yaml.safe_load(file) - api = yaml_data["api"]["eia"] + # $EIA_API_KEY wins over the yaml, matching workflow/Snakefile. + api = os.environ.get("EIA_API_KEY") + if not api: + with open("./../config/config.api.yaml") as file: + yaml_data = yaml.safe_load(file) + api = yaml_data["api"]["eia"] pipelines = InterconnectGasPipelineCapacity( year, diff --git a/workflow/scripts/eia.py b/workflow/scripts/eia.py index ace2f98ce..69fa96283 100644 --- a/workflow/scripts/eia.py +++ b/workflow/scripts/eia.py @@ -37,6 +37,7 @@ import logging import math +import os from abc import ABC, abstractmethod from typing import ClassVar @@ -1798,9 +1799,12 @@ def format_data(self, df: pd.DataFrame) -> pd.DataFrame: if __name__ == "__main__": - with open("./../config/config.api.yaml") as file: - yaml_data = yaml.safe_load(file) - api = yaml_data["api"]["eia"] + # $EIA_API_KEY wins over the yaml, matching workflow/Snakefile. + api = os.environ.get("EIA_API_KEY") + if not api: + with open("./../config/config.api.yaml") as file: + yaml_data = yaml.safe_load(file) + api = yaml_data["api"]["eia"] print( ElectricPowerData( "electric_power", From c11ed448344719e5e8124393b7dfd944c744e3ee Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 14:59:38 -0700 Subject: [PATCH 06/14] Validate the merged config against a JSON schema at parse time Config typos and out-of-range values had no early failure mode: an unknown key just sat in the merged dict while the rule that wanted it fell back to its `config_provider(..., default=...)` value, so the run completed with quietly wrong numbers. workflow/schemas/config.schema.yaml is checked by snakemake.utils.validate right after the configfile chain merges. Scope is deliberately partial: * the TOP LEVEL stays open - snakemake, the scenarios feature and user configs all inject keys there, and closing it would make legitimate extensions fatal; * well-bounded subtrees are closed (electricity, model_topology, run, scenario, clustering, solving, costs, conventional, lines, links, co2, dac, custom_files, snapshots, renewable_snapshots), which is where typos actually land; * enum knobs are enumerated from the inline comments in the configs; * set_default=False, and the schema declares no defaults, so it cannot become a second invisible source of configuration. The godeeep conditional requires renewable_land_access to be DECLARED and, if set, to be one of reference|limited|open. It cannot also require non-null: config.common.yaml ships it null while selecting godeeep, which build_renewable_profiles rejects at runtime - a pre-existing defect that is out of scope here and deliberately left for its own change. Verified: all five shipped configs validate; `electricity: retirment:` and each bad enum value fail with a clear ValidationError and exit 1. Tier A gains tests/static/test_config_schema.py, which replays the merge and the validation without spawning snakemake. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 1 + docs/source/config-configuration.md | 37 +++ tests/static/test_config_schema.py | 131 ++++++++ workflow/Snakefile | 10 +- workflow/schemas/config.schema.yaml | 493 ++++++++++++++++++++++++++++ 5 files changed, 671 insertions(+), 1 deletion(-) create mode 100644 tests/static/test_config_schema.py create mode 100644 workflow/schemas/config.schema.yaml diff --git a/CLAUDE.md b/CLAUDE.md index 18e4964fc..f4a16cfb3 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -94,6 +94,7 @@ Defined in `workflow/Snakefile`: - `repo_data/config/config.tutorial.yaml`, `config.test.yaml` — sparse overlays (only keys that differ from the base). `config.equivalence*.yaml` are deliberately self-contained because the Tier C harness replays them against a pinned upstream anchor that does not load the base. - `workflow/config/` is untracked and holds only per-user files, seeded by `init_pypsa_usa.sh`: `config.default.yaml` (your scenario starting point), `config.api.yaml`, `config.slurm.yaml`. Do not add layered configs back into it. - `policy_constraints/` CSVs are read straight from `repo_data/config/policy_constraints/`. +- The merged config is validated against `workflow/schemas/config.schema.yaml` at parse time (`snakemake.utils.validate`, `set_default=False`). The top level is open (snakemake/scenarios inject keys) but `electricity:`, `model_topology:`, `clustering:`, `solving:` etc. are closed, so a typo'd key fails loudly. Adding a config key means adding it to the schema. ## Things to know before changing rules diff --git a/docs/source/config-configuration.md b/docs/source/config-configuration.md index b9cb6acfa..cf1e9c0e9 100644 --- a/docs/source/config-configuration.md +++ b/docs/source/config-configuration.md @@ -1,6 +1,43 @@ (config)= # Configuration +## How configuration is layered + +`workflow/Snakefile` loads its whole base out of the tracked +`workflow/repo_data/config/` templates, in this order: + +| file | holds | +| --- | --- | +| `config.slurm.yaml` | HPC scheduler settings and the per-rule `walltime:` block | +| `config.common.yaml` | renewable/atlite plumbing that rarely changes per scenario | +| `config.plotting.yaml` | figure styling | +| `config.api.yaml` | API-key placeholders | +| `config.sector.yaml` | sector-coupling defaults | +| `config.default.yaml` | the scenario base — every user-facing knob, with defaults | + +Two optional per-user overlays follow (`config/config.api.yaml`, +`config/config.slurm.yaml`, seeded by `init_pypsa_usa.sh`), and whatever you +pass with `--configfile` is merged last. + +Because `config.default.yaml` is itself a loaded layer, **your scenario config is +an overlay, not a fork**: it only needs the keys it actually changes. Nested +mappings merge key by key; lists and scalars are replaced wholesale, so a list +you change must be restated in full. + +Every top-level key is owned by exactly one file — nothing is defined twice +across layers. + +## Validation + +The merged configuration is checked against `workflow/schemas/config.schema.yaml` +at parse time. The check covers the top-level structure, the enum-typed knobs +(`foresight`, `electricity: retirement`, `renewable: dataset`, clustering +algorithms, ATB scenario/model case, solver names, ...) and closes the +well-bounded sections such as `electricity:` and `model_topology:` against +unknown keys — so a misspelling like `retirment:` fails immediately instead of +silently falling back to a default several rules later. The top level itself +stays open, since Snakemake and the scenarios feature inject keys there. + (run_cf)= ## `run` diff --git a/tests/static/test_config_schema.py b/tests/static/test_config_schema.py new file mode 100644 index 000000000..d07f5e774 --- /dev/null +++ b/tests/static/test_config_schema.py @@ -0,0 +1,131 @@ +"""Tier A — the merged config of every shipped config must satisfy the schema. + +``workflow/Snakefile`` runs ``snakemake.utils.validate`` on the merged config +at parse time. This test replays the same merge and the same validation +without spawning snakemake, so a schema/config mismatch is caught in +milliseconds rather than at the top of a build. + +It also pins the schema's reason for existing: a typo'd key inside one of the +closed subtrees, and an out-of-range enum value, must both fail. +""" + +from __future__ import annotations + +import copy +from pathlib import Path + +import jsonschema +import pytest +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[2] +WORKFLOW_DIR = REPO_ROOT / "workflow" +CONFIG_DIR = WORKFLOW_DIR / "repo_data" / "config" +SCHEMA_PATH = WORKFLOW_DIR / "schemas" / "config.schema.yaml" + +# Must match the unconditional configfile: chain in workflow/Snakefile. +LAYERS = [ + "config.slurm.yaml", + "config.common.yaml", + "config.plotting.yaml", + "config.api.yaml", + "config.sector.yaml", + "config.default.yaml", +] + +SHIPPED_CONFIGS = [ + "config.default.yaml", + "config.tutorial.yaml", + "config.test.yaml", + "config.equivalence.yaml", + "config.equivalence-usa.yaml", +] + + +def _load(name: str) -> dict: + return yaml.safe_load((CONFIG_DIR / name).read_text()) or {} + + +def _merge(a: dict, b: dict) -> dict: + """Deep-merge b into a; b wins on conflict. + + Kept dependency-light (no snakemake import) like the other Tier A tests. + It differs from ``snakemake.utils.update_config`` in exactly one corner — + an empty mapping merged onto a null yields ``{}`` here and ``None`` there — + which only reaches ``model_topology.include``/``aggregate``, typed + ``[object, "null"]`` in the schema and therefore valid either way. + """ + out = dict(a) + for k, v in b.items(): + if k in out and isinstance(out[k], dict) and isinstance(v, dict): + out[k] = _merge(out[k], v) + else: + out[k] = v + return out + + +def _merged(user_config: str) -> dict: + merged: dict = {} + for name in LAYERS: + merged = _merge(merged, _load(name)) + if user_config not in LAYERS: + merged = _merge(merged, _load(user_config)) + return merged + + +@pytest.fixture(scope="module") +def schema() -> dict: + return yaml.safe_load(SCHEMA_PATH.read_text()) + + +@pytest.mark.fast +def test_schema_is_a_valid_json_schema(schema): + jsonschema.Draft7Validator.check_schema(schema) + + +@pytest.mark.fast +@pytest.mark.parametrize("config_name", SHIPPED_CONFIGS) +def test_shipped_configs_validate(config_name, schema): + jsonschema.validate(_merged(config_name), schema) + + +@pytest.mark.fast +def test_typo_in_closed_subtree_is_rejected(schema): + """A misspelled key under `electricity:` must not silently fall through.""" + cfg = copy.deepcopy(_merged("config.default.yaml")) + cfg["electricity"]["retirment"] = cfg["electricity"].pop("retirement") + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(cfg, schema) + + +@pytest.mark.fast +@pytest.mark.parametrize( + "path,bad_value", + [ + (("foresight",), "perfetc"), + (("electricity", "retirement"), "econmic"), + (("model_topology", "transmission_network"), "reads"), + (("clustering", "cluster_network", "algorithm"), "hac"), + (("solving", "solver", "name"), "gurbi"), + (("costs", "atb", "scenario"), "moderate"), + (("renewable_land_access",), "refrence"), + ], +) +def test_bad_enum_values_are_rejected(path, bad_value, schema): + cfg = copy.deepcopy(_merged("config.default.yaml")) + node = cfg + for key in path[:-1]: + node = node[key] + node[path[-1]] = bad_value + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(cfg, schema) + + +@pytest.mark.fast +def test_godeeep_requires_renewable_land_access_key(schema): + """The godeeep conditional must fire when the key is missing entirely.""" + cfg = copy.deepcopy(_merged("config.default.yaml")) + assert cfg["renewable"]["dataset"] == "godeeep" + cfg.pop("renewable_land_access") + with pytest.raises(jsonschema.ValidationError): + jsonschema.validate(cfg, schema) diff --git a/workflow/Snakefile b/workflow/Snakefile index be971dc97..68dceae22 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,4 +1,4 @@ -from snakemake.utils import min_version +from snakemake.utils import min_version, validate min_version("6.0") @@ -131,6 +131,14 @@ config.setdefault("api", {}) config["api"]["eia"] = os.environ.get("EIA_API_KEY") or config["api"].get("eia") +# Validate the fully merged config. Catches typo'd keys inside the closed +# subtrees and out-of-range enum values at parse time, instead of letting them +# fall through to a `config_provider(..., default=...)` fallback and surface as +# a wrong number several rules later. set_default=False keeps the schema from +# becoming a second, invisible source of configuration. +validate(config, "schemas/config.schema.yaml", set_default=False) + + run = config.get("run", {}) RDIR = run["name"] + "/" if run.get("name") else "" CDIR = RDIR if not run.get("shared_cutouts") else "" diff --git a/workflow/schemas/config.schema.yaml b/workflow/schemas/config.schema.yaml new file mode 100644 index 000000000..b390508f1 --- /dev/null +++ b/workflow/schemas/config.schema.yaml @@ -0,0 +1,493 @@ +# ==================================================================== +# PyPSA-USA — merged-configuration schema +# ==================================================================== +# Validated by `snakemake.utils.validate` in workflow/Snakefile once the +# whole configfile chain (including the user's --configfile) has merged. +# +# SCOPE, deliberately partial: +# * The TOP LEVEL allows additional properties. Snakemake, the scenarios +# feature and user configs all inject keys here, and closing it would +# turn every legitimate extension into a hard error. +# * Well-bounded subtrees ARE closed (`additionalProperties: false`) so a +# typo like `electricity: retirment:` fails loudly at parse time instead +# of silently falling through to a `config_provider(..., default=...)`. +# * Enum-typed knobs are enumerated from the inline comments in +# config.default.yaml / config.common.yaml. Free-form numeric leaves are +# mostly left untyped; depth can grow later. +# +# No `default:` keys anywhere: the Snakefile calls validate() with +# set_default=False, and injecting schema defaults into the merged config +# would make this file a silent second source of configuration. +# ==================================================================== + +$schema: http://json-schema.org/draft-07/schema# +type: object +additionalProperties: true + +required: +- run +- scenario +- foresight +- model_topology +- enable +- snapshots +- renewable_weather_years +- renewable_scenarios +- electricity +- conventional +- lines +- links +- clustering +- solving +- costs +- renewable +- atlite + +properties: + + run: + type: object + additionalProperties: false + required: [name] + properties: + name: {type: string} + disable_progressbar: {type: boolean} + shared_resources: {type: boolean} + shared_cutouts: {type: boolean} + validation: {type: boolean} + scenarios: + description: Optional multi-scenario driver; see rules/common.smk config_provider. + type: object + + scenario: + type: object + additionalProperties: false + required: [interconnect, planning_horizons, clusters, simpl, ll, opts, sector] + properties: + interconnect: + type: array + items: {enum: [usa, texas, western, eastern]} + planning_horizons: + type: array + items: {type: integer} + clusters: {type: array} + simpl: {type: array} + ll: {type: array} + opts: {type: array} + sector: {type: string} + scope: {enum: [urban, rural, total]} + + foresight: + enum: [perfect, myopic] + + model_topology: + type: object + additionalProperties: false + required: [transmission_network, topological_boundaries] + properties: + transmission_network: {enum: [reeds, tamu]} + topological_boundaries: {enum: [county, reeds_zone, state]} + interface_transmission_limits: {type: boolean} + trim: {type: boolean} + include: + description: > + Zone subset for footprint-scoped runs. null means "no filter"; see the + comment in config.default.yaml - consumers distinguish null from {}. + type: [object, "null"] + aggregate: + description: Pre-aggregation map; null means "no aggregation". + type: [object, "null"] + + enable: + type: object + additionalProperties: false + properties: + build_cutout: {type: boolean} + custom_busmap: {type: boolean} + + snapshots: + type: object + additionalProperties: false + required: [start, end, inclusive] + properties: + start: {type: string} + end: {type: string} + inclusive: {enum: [left, right, both, neither]} + + renewable_weather_years: + type: array + items: {type: integer} + + renewable_weather_years_by_horizon: + type: [object, "null"] + + renewable_scenarios: + type: array + minItems: 1 + items: + enum: [historical, rcp45cooler, rcp45hotter, rcp85cooler, rcp85hotter] + + renewable_scenario_years: + type: [array, "null"] + + renewable_snapshots: + type: object + additionalProperties: false + properties: + start_month: {type: integer, minimum: 1, maximum: 12} + start_day: {type: integer, minimum: 1, maximum: 31} + end_month: {type: integer, minimum: 1, maximum: 12} + end_day: {type: integer, minimum: 1, maximum: 31} + end_inclusive: {type: boolean} + + renewable: + description: Per-technology resource settings; owned by config.common.yaml. + type: object + additionalProperties: true + required: [dataset] + properties: + dataset: {enum: [atlite, godeeep]} + + renewable_land_access: + description: > + NREL exclusion-overlay variant. MUST be non-null when + renewable.dataset == godeeep - build_renewable_profiles raises otherwise. + The shipped templates leave it null (choose one in your own config), so + the conditional below requires the key to be DECLARED and, when set, to + carry a known value; it cannot also require non-null without failing the + shipped state. + enum: [reference, limited, open, null] + + apply_cec_basescreen: {type: boolean} + apply_boem_osw: {type: boolean} + godeeep_wind_height: {enum: ["_100m", "_125m"]} + + nrel_caps_reassign: + type: object + additionalProperties: false + properties: + enable: {type: boolean} + max_km: {type: number} + + pudl_path: {type: string} + + atlite: + type: object + additionalProperties: true + required: [cutouts, interconnects] + properties: + default_cutout: {type: string} + nprocesses: {type: integer} + show_progress: {type: boolean} + cutouts: {type: object} + interconnects: {type: object} + + offshore_shape: + type: object + additionalProperties: false + properties: + use: {enum: [ca_osw, eez]} + + offshore_network: + type: object + additionalProperties: false + properties: + bus_spacing: {type: number} + + electricity: + type: object + additionalProperties: false + required: [conventional_carriers, renewable_carriers, retirement, extendable_carriers] + properties: + conventional_carriers: {type: array, items: {type: string}} + renewable_carriers: {type: array, items: {type: string}} + retirement: {enum: [economic, technical]} + + extendable_carriers: + type: object + additionalProperties: false + properties: + Generator: {type: array, items: {type: string}} + StorageUnit: {type: array, items: {type: string}} + Store: {type: array, items: {type: string}} + Link: {type: array, items: {type: string}} + + SAFE_reservemargin: {type: number} + SAFE_regional_reservemargins: {type: string} + regional_Co2_limits: {type: string} + technology_capacity_targets: {type: string} + portfolio_standards: {type: string} + transmission_interface_limits: {type: string} + + operational_reserve: + type: object + additionalProperties: false + properties: + activate: {type: boolean} + epsilon_load: {type: number} + epsilon_vres: {type: number} + contingency: {type: number} + + erm: + type: object + + demand: + type: object + additionalProperties: false + properties: + profile: {enum: [efs, eia, eer]} + bus_allocation: {enum: [population, breakthrough]} + scenario: + type: object + additionalProperties: false + properties: + efs_case: {enum: [reference, medium, high]} + efs_speed: {enum: [slow, moderate, rapid]} + eer_file: {type: string} + aeo: {enum: [reference, high, low]} + + demand_response: + type: object + additionalProperties: false + properties: + shift: {type: number} + marginal_cost: {type: number} + by_carrier: {type: boolean} + + imports: + type: object + additionalProperties: false + properties: + enable: {type: boolean} + costs: {enum: [wholesale, carrier, float]} + co2_emissions: {type: number} + capacity_limit: {type: boolean} + volume_limit: {} + balancing_period: {enum: [day, week, month, year]} + + exports: + type: object + additionalProperties: false + properties: + enable: {type: boolean} + costs: {enum: [wholesale, carrier, float]} + capacity_limit: {type: boolean} + volume_limit: {} + balancing_period: {enum: [day, week, month, year]} + + conventional: + type: object + additionalProperties: false + properties: + unit_commitment: {type: boolean} + must_run: {type: boolean} + dynamic_fuel_price: + type: object + additionalProperties: false + properties: + enable: {type: boolean} + pudl: {type: boolean} + wholesale: {type: boolean} + + lines: + type: object + additionalProperties: false + required: [s_max_pu, types] + properties: + s_max_pu: {type: number} + s_nom_max: {type: number} + max_extension: {type: number} + length_factor: {type: number} + types: {type: object} + + links: + type: object + additionalProperties: false + properties: + p_max_pu: {type: number} + p_nom_max: {type: number} + max_extension: {type: number} + + co2: + type: object + additionalProperties: false + properties: + storage: {type: boolean} + network: + type: object + additionalProperties: false + properties: + enable: {type: boolean} + capital_cost: {type: number} + marginal_cost: {type: number} + lifetime: {type: number} + discount_rate: {type: number} + + dac: + type: object + additionalProperties: false + properties: + enable: {type: boolean} + granularity: {enum: [node, state, nation]} + capital_cost: {type: number} + electricity_input: {type: number} + lifetime: {type: number} + discount_rate: {type: number} + + costs: + type: object + additionalProperties: false + required: [atb, aeo] + properties: + atb: + type: object + additionalProperties: false + required: [model_case, scenario] + properties: + model_case: {enum: ["Market", "R&D"]} + scenario: {enum: ["Advanced", "Conservative", "Moderate"]} + overrides: {type: [object, "null"]} + aeo: + type: object + additionalProperties: false + properties: + scenario: {enum: [reference, high, low]} + social_discount_rate: {type: number} + ng_fuel_year: {type: integer} + emission_prices: + type: object + additionalProperties: false + properties: + enable: {type: boolean} + co2: {type: number} + co2_monthly_prices: {type: boolean} + ptc_modifier: {type: [object, "null"]} + itc_modifier: {type: [object, "null"]} + min_year: {type: [object, "null"]} + max_growth: {type: [object, "null"]} + + clustering: + type: object + additionalProperties: false + required: [simplify_network, cluster_network] + properties: + simplify_network: + type: object + additionalProperties: false + properties: + weighting_strategy: {enum: [population, demand-capacity]} + algorithm: {enum: [kmeans, modularity]} + feature: {type: string} + cluster_network: + type: object + additionalProperties: false + properties: + weighting_strategy: {enum: [population, demand-capacity]} + algorithm: {enum: [kmeans, modularity]} + exclude_carriers: {type: array} + consider_efficiency_classes: {type: boolean} + feature: {type: string} + aggregation_strategies: {type: object} + temporal: + type: object + additionalProperties: false + properties: + resolution_elec: {type: [integer, boolean]} + resolution_sector: {type: [integer, boolean]} + + focus_weights: + description: > + Per-region clustering weights. null (not {}) means uniform - cluster_network + distinguishes the two, so leave it null rather than emptying it. + type: [object, "null"] + + solving: + type: object + additionalProperties: false + required: [options, solver, solver_options] + properties: + tmpdir: {type: string} + mem: {type: number} + walltime: {type: string} + options: + type: object + additionalProperties: false + properties: + load_shedding: {type: boolean} + clip_p_max_pu: {type: number} + noisy_costs: {type: boolean} + skip_iterations: {type: boolean} + rolling_horizon: {type: boolean} + seed: {type: integer} + track_iterations: {type: boolean} + min_iterations: {type: integer} + max_iterations: {type: integer} + transmission_losses: {type: integer} + linearized_unit_commitment: {type: boolean} + horizon: {type: integer} + assign_all_duals: {type: boolean} + snapshot_portion: {type: [number, "null"]} + overlap: {type: integer} + solver: + type: object + additionalProperties: false + required: [name, options] + properties: + name: {enum: [gurobi, highs, cplex, cbc, glpk]} + options: {type: string} + solver_options: {type: object} + + walltime: + description: Per-rule scheduler wall times; owned by config.slurm.yaml. + type: object + + custom_files: + type: object + additionalProperties: false + properties: + activate: {type: boolean} + files_path: {type: string} + network_name: {type: string} + + ucap: + type: object + additionalProperties: false + properties: + enable: {type: boolean} + forced_outage_rates: {type: object} + + api: + type: object + properties: + eia: {type: ["string", "null"]} + +allOf: +# The selected solver's options block must exist in solver_options. +- if: + properties: + solving: + properties: + solver: + required: [options] + required: [solver] + required: [solving] + then: + properties: + solving: + properties: + solver_options: {type: object, minProperties: 1} + +# GODEEEP needs an NREL land-access overlay. See the note on the property +# itself: the key must be declared and, when set, carry a known value. +- if: + properties: + renewable: + properties: + dataset: {const: godeeep} + required: [dataset] + required: [renewable] + then: + required: [renewable_land_access] + properties: + renewable_land_access: + enum: [reference, limited, open, null] From 1136c888908804b1e3b9b3089bee7fe1a7f02e62 Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 15:01:33 -0700 Subject: [PATCH 07/14] Document the load-bearing nulls; make model_topology.aggregate an empty map Bare nulls in the config read as "unfinished" but several of them are meaningful, and blanket-converting them to {} would change behaviour. Each was checked against its consumer: model_topology.include STAYS NULL. build_base_network reads "not None" as "a footprint filter was requested" and then asserts topological_boundaries != 'state'; {} would trip that assertion on every state-resolution run. focus_weights STAYS NULL. cluster_network branches on `is not None`, so {} would take the custom-weights path, re-normalise every region's factor and log a misleading warning. costs.max_growth stays null; {} is equivalent (the consumer short-circuits on None and would otherwise iterate nothing), so there is nothing to gain by changing it. model_topology.aggregate BECOMES {}. Its only consumer is a truthiness check in cluster_network (the parameter is not even referenced inside convert_to_transport), so {} and None behave identically - and {} is what config.test.yaml and both equivalence configs merged to before config.default.yaml sat underneath them, because snakemake's update_config maps an empty mapping onto a null parent as null. Net effect on the merged config: config.test.yaml and both equivalence configs go back to their pre-refactor value ({}), and config.default.yaml/tutorial move from None to {} with identical behaviour. Also documents, without changing, that config.common.yaml ships renewable_land_access null while selecting the godeeep dataset - build_renewable_profiles rejects that combination, so the shipped templates cannot build profiles until the user picks a value. Choosing one here would add the NREL exclusion artifacts to every DAG and move the numbers, so it is left for its own change. Co-Authored-By: Claude Fable 5 --- workflow/repo_data/config/config.common.yaml | 8 ++++++++ workflow/repo_data/config/config.default.yaml | 17 ++++++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/workflow/repo_data/config/config.common.yaml b/workflow/repo_data/config/config.common.yaml index a972ad8bd..89a858ada 100644 --- a/workflow/repo_data/config/config.common.yaml +++ b/workflow/repo_data/config/config.common.yaml @@ -162,6 +162,14 @@ renewable: # per-bus supply-curve capacities. The variables here pick which # overlay variant gets loaded. # docs : NREL_EXCLUSION +# NULL IS NOT A WORKING VALUE. `renewable.dataset` defaults to godeeep, and +# build_renewable_profiles raises "renewable_land_access must be set" when this +# is null - so the shipped templates cannot build renewable profiles until you +# choose one in your own config. The schema requires the key to be declared and +# to carry a known value when set; it cannot require non-null without failing +# the shipped state. Left as-is here deliberately: picking a default would +# change every run's DAG (it pulls in the NREL exclusion artifacts) and the +# numbers that come out of it. renewable_land_access: # REQUIRED for godeeep; one of: reference | limited | open apply_cec_basescreen: false # overlay CEC Wind/Solar BaseScreen on onwind+solar (CA-only) apply_boem_osw: false # overlay BOEM offshore-wind planning areas on offwind + offwind_floating diff --git a/workflow/repo_data/config/config.default.yaml b/workflow/repo_data/config/config.default.yaml index edb64a278..e0697f0a0 100644 --- a/workflow/repo_data/config/config.default.yaml +++ b/workflow/repo_data/config/config.default.yaml @@ -81,13 +81,17 @@ model_topology: transmission_network: 'reeds' # transmission dataset; reeds | tamu topological_boundaries: 'reeds_zone' # post-cluster zone type; county | reeds_zone | state interface_transmission_limits: false # apply NARIS2024 inter-region transfer caps (requires reeds backbone) + # KEEP THIS NULL, not {}. build_base_network treats "not None" as "the user + # asked for a footprint filter" and then asserts that topological_boundaries + # is not 'state'; an empty mapping would trip that assertion on every + # state-resolution run even though no filtering was requested. include: # subset the network to specific zones (mixed zone types NOT supported) # reeds_zone: [] # restrict by REeDS zone ids # reeds_state: ['CA'] # restrict by state code(s) # reeds_ba: [] # restrict by balancing authority - aggregate: # pre-aggregate buses; eligible keys: reeds_zone | trans_reg - # trans_grp: [] - # reeds_zone: [] + aggregate: {} # pre-aggregate buses; eligible keys: reeds_zone | trans_reg + # trans_grp: [] # (empty mapping = no aggregation; the only + # reeds_zone: [] # consumer is a truthiness check in cluster_network) # ==================================================================== @@ -333,6 +337,9 @@ costs: min_year: # earliest commercial-availability year per carrier; carrier excluded before this year hydrogen_ct: 2040 + # null and {} are equivalent here (add_extra_components short-circuits on + # None and would otherwise iterate an empty mapping); null is kept as the + # "nothing configured" spelling used by the other optional maps. max_growth: # per-carrier annual build-rate caps; {carrier: {base: MW, rate: fraction}} @@ -370,6 +377,10 @@ clustering: resolution_elec: false # int H or false = native hourly resolution_sector: false # int H or false = native hourly +# KEEP THIS NULL, not {}. cluster_network branches on `is not None`, and an +# empty mapping would send it down the custom-weights path: it re-normalises +# every region's factor and logs "using custom focus weights", for a result +# that is only float-equal to the uniform path by luck. focus_weights: # OPTIONAL: per-region weights for the clustering objective; null = uniform From 34b5163f3db320b1ad526a3c2496a481e10d0bb3 Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 15:03:48 -0700 Subject: [PATCH 08/14] Apply ruff-format and mark the two new config-layout guards fast Formatting only, plus @pytest.mark.fast on test_init_copies_only_user_owned_files and test_snakefile_reads_layered_base_from_repo_data so CI's `pytest -m fast` job actually runs them - without the marker they would never execute in CI. Co-Authored-By: Claude Fable 5 --- tests/docs/test_docs_config.py | 7 ++++--- tests/equivalence/paths.py | 4 +--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/docs/test_docs_config.py b/tests/docs/test_docs_config.py index 412e77101..ee1be9969 100644 --- a/tests/docs/test_docs_config.py +++ b/tests/docs/test_docs_config.py @@ -228,8 +228,9 @@ def _init_script() -> tuple[Path, Path, set[str]]: return REPO_ROOT / templates.group(1), REPO_ROOT / destination.group(1), names +@pytest.mark.fast def test_init_copies_only_user_owned_files(): - """init must seed exactly the per-user files, and each must exist as a template. + """Init must seed exactly the per-user files, each backed by a template. Everything else under ``workflow/repo_data/config/`` is loaded by ``workflow/Snakefile`` straight from the tracked tree, so copying it into @@ -252,6 +253,7 @@ def test_init_copies_only_user_owned_files(): }, f"unexpected per-user file in init_pypsa_usa.sh: {sorted(user_files)}" +@pytest.mark.fast def test_snakefile_reads_layered_base_from_repo_data(): """The layered base must be read from the tracked templates, not from copies.""" snakefile = (REPO_ROOT / "workflow" / "Snakefile").read_text(encoding="utf-8") @@ -259,6 +261,5 @@ def test_snakefile_reads_layered_base_from_repo_data(): assert layered, "no unconditional configfile: directives found in workflow/Snakefile" strays = [p for p in layered if not p.startswith("repo_data/config/")] assert not strays, ( - "these layered configfiles are still read from the per-user config/ copy " - f"and will drift: {strays}" + f"these layered configfiles are still read from the per-user config/ copy and will drift: {strays}" ) diff --git a/tests/equivalence/paths.py b/tests/equivalence/paths.py index 79bc3e582..af7894472 100644 --- a/tests/equivalence/paths.py +++ b/tests/equivalence/paths.py @@ -22,9 +22,7 @@ RUN = "equivalence" INTERCONNECT = os.environ.get("EQ_INTERCONNECT", "western") UNTIL = os.environ.get("EQ_UNTIL", "") # 'assembled' = stop pairs at the assembled stage -_CONFIG_NAME = ( - "config.equivalence.yaml" if INTERCONNECT == "western" else f"config.equivalence-{INTERCONNECT}.yaml" -) +_CONFIG_NAME = "config.equivalence.yaml" if INTERCONNECT == "western" else f"config.equivalence-{INTERCONNECT}.yaml" # Candidate side reads the tracked template directly (its Snakefile no longer # needs a config/ copy). The anchor is a pinned upstream checkout whose # Snakefile still expects everything under config/, and build.py copies the From b2a29c73092576c150930fb496d844bc54f9dc4a Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 14:24:53 -0700 Subject: [PATCH 09/14] CLAUDE.md: require agents to reference develop, not master MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The working tree often sits on master, which lags develop badly; an agent analyzing "current state" from it recently reported config issues develop had already fixed. Make the branch policy explicit: all model work — changes, reviews, audits, analysis — must reference origin/develop, and feature branches/PRs base on develop. Co-Authored-By: Claude Fable 5 --- CLAUDE.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index f4a16cfb3..15d14613c 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,6 +6,10 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co PyPSA-USA is a Snakemake-orchestrated PyPSA workflow for capacity expansion, production-cost simulation, and power-flow analysis of the US bulk transmission system. Configuration is layered YAML; intermediate and final artifacts are netCDF/CSV/GeoJSON files produced by rules in `workflow/rules/*.smk` and Python scripts in `workflow/scripts/`. +## Branch policy: work against `develop`, not `master` + +`develop` is the integration branch and is often well ahead of `master`. Any work on the model — code changes, config changes, reviews, audits, or analysis of "current state" — MUST reference `origin/develop`, not `master` or whatever the working tree happens to have checked out. Before drawing conclusions about how something works or proposing changes, run `git fetch origin develop` and check the file's state on `origin/develop` (`git show origin/develop:` or check out a branch based on it). Base all feature branches and PRs on `develop`; never target `master` directly. + ## Running the workflow **All `snakemake` invocations run from `workflow/`** — `cd workflow/` first. `workflow/Snakefile` auto-loads the whole layered base out of the tracked templates: `repo_data/config/config.{slurm,common,plotting,api,sector,default}.yaml`, then the optional per-user overlays `config/config.api.yaml` and `config/config.slurm.yaml`, then whatever is passed via `--configfile`. Because `config.default.yaml` is a loaded layer, a scenario config is a sparse **overlay** — it only needs the keys it changes (nested mappings merge; lists and scalars are replaced wholesale). From 314414707f22216fa892e234a6cd2b9987ec06f8 Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 15:15:40 -0700 Subject: [PATCH 10/14] docs: finish the config-reorganization sweep The reorganization commits updated the prose pages but left three statements behind that the new loading order makes false: - The configtables still pointed the five `electricity:` policy-CSV paths (and the `REM` opts row) at `config/policy_constraints/`. The rules now read those CSVs straight out of `repo_data/config/policy_constraints/`; there is no copy step. - about-usage.md's troubleshooting note claimed every invocation must pass `--configfile` and that omitting it dies with `KeyError: 'scenario'`. `config.default.yaml` is now a loaded layer, so a bare `snakemake -n` resolves and builds the shipped defaults (verified by dry-run). Replaced with what actually happens. - The same page told users the `--configfile` in run_slurm.sh defaults to `config/config.default.yaml`; the script has always shipped a personal path. Stated it as the example it is, and named the `--cluster-config config/config.slurm.yaml` argument next to it. Also completes two things the sweep asked for and the pages only half said: about-usage.md now spells out the two ways to author a run (copy the seeded default, or write a bare overlay) and the wholesale-list replacement rule; config-configuration.md notes that $EIA_API_KEY wins over config/config.api.yaml. release-notes.md gains the four user-visible migration items that were missing from a section that promises to list them: the config.slurm.yaml rename, sparse overlays, schema validation at parse time, and the API-key environment variable. Nothing else in docs/source needed changing: the per-section YAML is pulled in by literalinclude, so the in-file comments documenting the load-bearing nulls render on their own, and no configtable attributes a key to a file that the one-file-per-key split moved. pytest tests/docs: 40 passed (unchanged). Sphinx -n build: no warnings. Co-Authored-By: Claude Fable 5 --- docs/source/about-usage.md | 12 ++++++++---- docs/source/config-configuration.md | 4 ++++ docs/source/configtables/electricity.csv | 10 +++++----- docs/source/configtables/opts.csv | 2 +- docs/source/release-notes.md | 15 +++++++++++++++ 5 files changed, 33 insertions(+), 10 deletions(-) diff --git a/docs/source/about-usage.md b/docs/source/about-usage.md index 50c9aa4b9..3c2e6fe10 100644 --- a/docs/source/about-usage.md +++ b/docs/source/about-usage.md @@ -13,7 +13,10 @@ default configuration in `config/config.default.yaml` (seeded from the tracked t Your config file only needs to carry the keys you actually change: the workflow always loads `repo_data/config/config.default.yaml` and the other layered files underneath it, and -your `--configfile` is merged on top. +your `--configfile` is merged on top. So there are two equally valid ways to set up a run — +copy the seeded `config/config.default.yaml` to `config/config..yaml` and edit it, +or write a short file holding only your overrides. Nested mappings merge key by key, but +lists and scalars are **replaced wholesale**, so any list you change must be restated in full. You can find more information on each configuration setting on the [configurations page](https://pypsa-usa.readthedocs.io/en/latest/config-configuration.html). @@ -53,7 +56,7 @@ snakemake data_model -j1 --configfile config/config.default.yaml If you are running the workflow on an High-Performance Compute (HPC) cluster, you will first need to update the configuration settings in `workflow/config/config.slurm.yaml` (seeded by `init_pypsa_usa.sh`). Update the account, partition, email, and chdir fields to match the information of your institutions cluster. -Next, identify the name of the configuration file you would like to run by editing the `run_slurm.sh` script. The default value is the `--configfile config/config.default.yaml`. +Next, identify the name of the configuration file you would like to run by editing the `--configfile` argument in the `run_slurm.sh` script; the path shipped in the script is only an example. The script also passes `--cluster-config config/config.slurm.yaml`, which is what resolves the `{cluster.*}` placeholders in its `sbatch` command line. To run, open a terminal within a login node of your cluster and run the script included in the `workflow` directory: @@ -107,6 +110,7 @@ uv run snakemake -j4 -R build_shapes --until build_base_network --configfile con where `build_shapes` is forced to run, and `build_base_network` is the last rule you would like to run. ```{note} -Every `snakemake` invocation must include `--configfile` (the Snakefile does not set a -default configuration file). Omitting it fails with `KeyError: 'scenario'`. +`--configfile` is optional: the Snakefile loads `repo_data/config/config.default.yaml` as +the base layer, so omitting it simply runs the shipped defaults under `run: name: "Default"`. +Pass `--configfile` to select your own scenario and give it its own `run: name:`. ``` diff --git a/docs/source/config-configuration.md b/docs/source/config-configuration.md index cf1e9c0e9..3499c3ae1 100644 --- a/docs/source/config-configuration.md +++ b/docs/source/config-configuration.md @@ -27,6 +27,10 @@ you change must be restated in full. Every top-level key is owned by exactly one file — nothing is defined twice across layers. +The one value that does not come from a file at all is the EIA API key: `workflow/Snakefile` +reads `$EIA_API_KEY` from the environment and it takes precedence over `api: eia:` in +`config/config.api.yaml`, which keeps the secret out of the checkout entirely. + ## Validation The merged configuration is checked against `workflow/schemas/config.schema.yaml` diff --git a/docs/source/configtables/electricity.csv b/docs/source/configtables/electricity.csv index af570d630..792503da4 100644 --- a/docs/source/configtables/electricity.csv +++ b/docs/source/configtables/electricity.csv @@ -10,7 +10,7 @@ extendable_carriers:,,,Capacity-expansion candidates. Carriers listed here get e -- Link,--,"e.g. {HVDC}",Adds extendable links at every connection where there are lines or HVDC links with zero initial capacity. ,,, SAFE_reservemargin,p.u.,float,"System-wide planning reserve margin as a fraction of peak load. Reserved setting — not currently enforced by an active constraint; the ``ERM`` option is the active resource-adequacy mechanism." -SAFE_regional_reservemargins,--,path,"CSV of per-region planning reserve margins (``config/policy_constraints/SAFE_regional_prm.csv``). Reserved setting — not currently enforced by an active constraint." +SAFE_regional_reservemargins,--,path,"CSV of per-region planning reserve margins (``repo_data/config/policy_constraints/SAFE_regional_prm.csv``). Reserved setting — not currently enforced by an active constraint." ,,, operational_reserve:,,,Settings for reserve requirements following `GenX `_ -- activate,bool,true or false,Whether to take operational reserve requirements into account during optimisation @@ -21,10 +21,10 @@ operational_reserve:,,,Settings for reserve requirements following `GenX ,--,float,"Reserve margin as a fraction (e.g., 0.15 for 15%). Use ``all`` for all regions, or specify region codes (state, interconnect, NERC region, or ReEDS zone). Defaults to ``all: 0.15`` if not specified." ,,, -regional_Co2_limits,--,path,"CSV of per-region CO2 caps in tCO2/yr (``config/policy_constraints/regional_Co2_limits.csv``). Enforced when the ``REM`` keyword is present in the ``{opts}`` wildcard." -technology_capacity_targets,--,path,"CSV of forced minimum/maximum capacity builds by technology and region (``config/policy_constraints/technology_capacity_targets.csv``). Enforced when the ``TCT`` keyword is present in the ``{opts}`` wildcard." -portfolio_standards,--,path,"CSV of RPS/CES clean-energy fractions by region (``config/policy_constraints/portfolio_standards.csv``). Enforced when the ``RPS`` keyword is present in the ``{opts}`` wildcard (covers both RPS and CES targets)." -transmission_interface_limits,--,path,"CSV of MW limits on flows across inter-regional transmission interfaces (``config/policy_constraints/transmission_interface_limits.csv``), paired with ``model_topology: interface_transmission_limits``. Reserved setting — not currently consumed by the workflow." +regional_Co2_limits,--,path,"CSV of per-region CO2 caps in tCO2/yr (``repo_data/config/policy_constraints/regional_Co2_limits.csv``). Enforced when the ``REM`` keyword is present in the ``{opts}`` wildcard." +technology_capacity_targets,--,path,"CSV of forced minimum/maximum capacity builds by technology and region (``repo_data/config/policy_constraints/technology_capacity_targets.csv``). Enforced when the ``TCT`` keyword is present in the ``{opts}`` wildcard." +portfolio_standards,--,path,"CSV of RPS/CES clean-energy fractions by region (``repo_data/config/policy_constraints/portfolio_standards.csv``). Enforced when the ``RPS`` keyword is present in the ``{opts}`` wildcard (covers both RPS and CES targets)." +transmission_interface_limits,--,path,"CSV of MW limits on flows across inter-regional transmission interfaces (``repo_data/config/policy_constraints/transmission_interface_limits.csv``), paired with ``model_topology: interface_transmission_limits``. Reserved setting — not currently consumed by the workflow." ,,, co2limit_enable,bool,true or false,"Switch to activate the system-wide CO2 cap below. Optional; defaults to false when unset. Can also be set via the ``Co2L`` keyword in the ``{opts}`` wildcard." co2limit,:math:`t_{CO_2}/a`,float,"System-wide cap on annual CO2 emissions, added as a global constraint in ``prepare_network``. Only applied when ``co2limit_enable`` is true." diff --git a/docs/source/configtables/opts.csv b/docs/source/configtables/opts.csv index 85aa48510..176012b1f 100644 --- a/docs/source/configtables/opts.csv +++ b/docs/source/configtables/opts.csv @@ -6,7 +6,7 @@ Trigger, Description, Definition, Status ``Ep``; e.g. ``Ep`` or ``Ep50``,"Add a carbon-dioxide price to the marginal cost of emitting generators and storage units, weighted by emission intensity over efficiency (sets ``costs: emission_prices: enable: true``). An appended number sets the price in $/tCO2; otherwise ``costs: emission_prices: co2`` is used.","``prepare_network``: `add_emission_prices() `_", In active use ``Ept``,"Request time-varying monthly carbon-dioxide prices (sets ``costs: emission_prices: co2_monthly_prices: true``). No workflow step currently consumes this setting; because the token also matches the ``Ep`` parser, its net effect equals plain ``Ep`` (static price from ``costs: emission_prices: co2``).","Parsed in `update_config_from_wildcards() `_", Parsed; monthly pricing not implemented (falls back to static ``Ep``) ``RPS``,"Add Renewable Portfolio Standard (RPS) and Clean Energy Standard (CES) constraints enforcing minimum shares of generation (MWh) from eligible carriers per REC trading zone and planning horizon. Merges state targets from ``electricity: portfolio_standards`` (CSV) with ReEDS RPS/CES trajectories. Regions may be ReEDS zones, state codes, interconnects, or NERC regions. See :doc:`/model-constraints`.","``solve_network``: `add_RPS_constraints() `_", In active use -``REM``,"Add regional carbon-dioxide emission limits per planning horizon, configured via the CSV at ``electricity: regional_Co2_limits`` (default ``config/policy_constraints/regional_Co2_limits.csv``). In sector studies this token instead dispatches sector-specific emission limits. See :doc:`/model-constraints`.","``solve_network``: `add_regional_co2limit() `_", In active use +``REM``,"Add regional carbon-dioxide emission limits per planning horizon, configured via the CSV at ``electricity: regional_Co2_limits`` (default ``repo_data/config/policy_constraints/regional_Co2_limits.csv``). In sector studies this token instead dispatches sector-specific emission limits. See :doc:`/model-constraints`.","``solve_network``: `add_regional_co2limit() `_", In active use ``ERM``,"Add Energy Reserve Margin (ERM) constraints requiring energy-backed firm capacity above demand in every snapshot for defined regions. Generators are credited at their availability; storage and transmission contribute through shadow reserve dispatch. Configured via ``electricity: erm``; defaults to a 15% margin for all regions. See :doc:`/model-constraints`.","``solve_network``: `add_ERM_constraints() `_", In active use ``TCT``,"Add minimum or maximum nominal-capacity targets per carrier group, region, and planning horizon, configured via the CSV at ``electricity: technology_capacity_targets``. In myopic runs, ``max = 0`` targets additionally force retirement of existing capacity. See :doc:`/model-constraints`.","``solve_network``: `add_technology_capacity_target_constraints() `_", In active use ``+{p|e|c|m}``; e.g. ``solar+c0.5``,"Scale an attribute of a single carrier: ``p`` = ``p_nom_max``, ``e`` = ``e_nom_max``, ``c`` = ``capital_cost``, ``m`` = ``marginal_cost``. Parsed into ``adjustments: electricity``, which no workflow step currently applies to the network.","Parsed in `update_config_from_wildcards() `_", Parsed; currently without effect (``adjustments`` has no consumer) diff --git a/docs/source/release-notes.md b/docs/source/release-notes.md index 6dd5fec63..b46dc9237 100644 --- a/docs/source/release-notes.md +++ b/docs/source/release-notes.md @@ -34,6 +34,21 @@ changes you will notice: base from there; `workflow/config/` is untracked and holds only the per-user files (`config.api.yaml`, `config.slurm.yaml`, your own scenario configs) seeded by `init_pypsa_usa.sh`. +- **`config.cluster.yaml` was renamed `config.slurm.yaml`.** The old name collided with the + `{clusters}` wildcard and the `clustering:` section while only ever describing the job + scheduler. Its dead per-rule `{rule}: {walltime: ...}` blocks are gone; the single + top-level `walltime:` block is the one the rules actually read. Rename your copy under + `workflow/config/` to keep your account and partition settings. +- **Scenario configs are overlays, not forks.** `config.default.yaml` is itself a loaded + layer, so a `--configfile` only needs the keys it changes, and every top-level key is now + owned by exactly one layered file. A full copy still works — it simply overrides every key + it repeats. See {doc}`config-configuration`. +- **The merged configuration is validated** against `workflow/schemas/config.schema.yaml` at + parse time. A misspelled key inside a closed section (`electricity:`, `model_topology:`, + `clustering:`, `solving:`, ...) now fails immediately instead of silently falling back to a + default several rules later. +- **The EIA API key can be supplied as `$EIA_API_KEY`**, which takes precedence over + `config/config.api.yaml` and keeps the key out of your files entirely. ### Correctness fixes validated by an equivalence harness From 82bbe176f6ee4aa2f23fb5a33e58da9fe16e9e4b Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 15:13:10 -0700 Subject: [PATCH 11/14] Default renewable_land_access to 'reference'; fix Tier B CI cache path The shipped templates carried renewable.dataset: godeeep with renewable_land_access null, a pair build_renewable_profiles hard-rejects - so no shipped config could build renewable profiles from a clean checkout. Same drift class as #793: the maintainer's untracked config was papering over a broken tracked default. Set 'reference', NREL's central land-access assumption and the value the equivalence harness validates against (both equivalence configs already pin it, so their merged configs are unchanged). Adds retrieve_nrel_exclusion_artifact jobs to every godeeep DAG (6 for the tutorial). With a working default in place, tighten the schema's godeeep conditional to require non-null (null stays valid for dataset: atlite). This went unnoticed because CI's e2e job caches data/cutouts at the repo root while tests/integration/conftest.py checks workflow/data and workflow/cutouts, so Tier B silently skipped every run. Point the cache at the workflow/ paths and bump DATA_CACHE_NUMBER so stale root-level archives are not restored. Co-Authored-By: Claude Fable 5 --- .github/workflows/main.yml | 6 +++--- workflow/repo_data/config/config.common.yaml | 15 ++++++--------- workflow/schemas/config.schema.yaml | 14 ++++++-------- 3 files changed, 15 insertions(+), 20 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cd5ab0b11..107a3e16f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,7 @@ on: - cron: "0 5 * * TUE" # weekly upstream-master regression env: - DATA_CACHE_NUMBER: 2 + DATA_CACHE_NUMBER: 3 jobs: fast-tests: @@ -59,8 +59,8 @@ jobs: uses: actions/cache@v4 with: path: | - data - cutouts + workflow/data + workflow/cutouts key: data-cutouts-${{ env.WEEK }}-${{ env.DATA_CACHE_NUMBER }} - name: Install test extras into the conda env run: pip install -e '.[test]' diff --git a/workflow/repo_data/config/config.common.yaml b/workflow/repo_data/config/config.common.yaml index 89a858ada..bec60d4ff 100644 --- a/workflow/repo_data/config/config.common.yaml +++ b/workflow/repo_data/config/config.common.yaml @@ -162,15 +162,12 @@ renewable: # per-bus supply-curve capacities. The variables here pick which # overlay variant gets loaded. # docs : NREL_EXCLUSION -# NULL IS NOT A WORKING VALUE. `renewable.dataset` defaults to godeeep, and -# build_renewable_profiles raises "renewable_land_access must be set" when this -# is null - so the shipped templates cannot build renewable profiles until you -# choose one in your own config. The schema requires the key to be declared and -# to carry a known value when set; it cannot require non-null without failing -# the shipped state. Left as-is here deliberately: picking a default would -# change every run's DAG (it pulls in the NREL exclusion artifacts) and the -# numbers that come out of it. -renewable_land_access: # REQUIRED for godeeep; one of: reference | limited | open +# REQUIRED (non-null) whenever `renewable.dataset` is godeeep - the schema and +# build_renewable_profiles both enforce it. `reference` is NREL's central +# land-access assumption and the value the equivalence harness validates +# against. Note this key drives the DAG: any godeeep run pulls the NREL +# exclusion artifacts (retrieve_nrel_exclusion_artifact jobs). +renewable_land_access: reference # reference | limited | open (null allowed only with dataset: atlite) apply_cec_basescreen: false # overlay CEC Wind/Solar BaseScreen on onwind+solar (CA-only) apply_boem_osw: false # overlay BOEM offshore-wind planning areas on offwind + offwind_floating godeeep_wind_height: "_100m" # raw GODEEEP wind-CF hub height; "_100m" | "_125m" diff --git a/workflow/schemas/config.schema.yaml b/workflow/schemas/config.schema.yaml index b390508f1..cebc4ffd6 100644 --- a/workflow/schemas/config.schema.yaml +++ b/workflow/schemas/config.schema.yaml @@ -151,11 +151,9 @@ properties: renewable_land_access: description: > NREL exclusion-overlay variant. MUST be non-null when - renewable.dataset == godeeep - build_renewable_profiles raises otherwise. - The shipped templates leave it null (choose one in your own config), so - the conditional below requires the key to be DECLARED and, when set, to - carry a known value; it cannot also require non-null without failing the - shipped state. + renewable.dataset == godeeep - build_renewable_profiles raises otherwise, + and the conditional below enforces it at config-load time. Null is only + valid with dataset: atlite (the legacy availability path). enum: [reference, limited, open, null] apply_cec_basescreen: {type: boolean} @@ -477,8 +475,8 @@ allOf: properties: solver_options: {type: object, minProperties: 1} -# GODEEEP needs an NREL land-access overlay. See the note on the property -# itself: the key must be declared and, when set, carry a known value. +# GODEEEP needs an NREL land-access overlay: the key must be declared and +# non-null (null is only valid with dataset: atlite). - if: properties: renewable: @@ -490,4 +488,4 @@ allOf: required: [renewable_land_access] properties: renewable_land_access: - enum: [reference, limited, open, null] + enum: [reference, limited, open] From 2a2f9e2b62e30d561d88236ac9fe87667ab5e05a Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 15:19:55 -0700 Subject: [PATCH 12/14] docs: reflect the reference land-access default Update the nrel_exclusion configtable (default column, null now atlite-only, correct the restrictiveness ordering: open > reference > limited) and add the new default to the release-notes migration list. Co-Authored-By: Claude Fable 5 --- docs/source/configtables/nrel_exclusion.csv | 2 +- docs/source/release-notes.md | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/source/configtables/nrel_exclusion.csv b/docs/source/configtables/nrel_exclusion.csv index 0de27efda..91d9bc267 100644 --- a/docs/source/configtables/nrel_exclusion.csv +++ b/docs/source/configtables/nrel_exclusion.csv @@ -1,5 +1,5 @@ ,Unit,Values,Description -renewable_land_access,--,"One of {``reference``, ``limited``, ``open``}; ``null`` to disable","NREL reV access scenario applied as availability weighting to GODEEEP capacity factors. Required when ``renewable: dataset: godeeep`` — otherwise the workflow raises. ``reference`` is the least restrictive; ``limited`` is closest to the legacy CORINE mask; ``open`` is the most permissive (sensitivity studies). Selects the avail/caps file pair downloaded from Zenodo." +renewable_land_access,``reference``,"One of {``reference``, ``limited``, ``open``}; ``null`` only valid with ``dataset: atlite``","NREL reV access scenario applied as availability weighting to GODEEEP capacity factors. Required non-null when ``renewable: dataset: godeeep`` — the schema and the workflow both raise otherwise. ``reference`` is NREL's central land-access assumption (the default); ``limited`` is the most restrictive; ``open`` is the most permissive (sensitivity studies). Selects the avail/caps file pair downloaded from Zenodo." apply_cec_basescreen,--,bool,"If true, overlay the California Energy Commission Wind/Solar BaseScreen on the NREL exclusion raster for onwind and solar (CA-only — outside California the variant equals the base). Selects the ``_cec`` file variants. No effect on offshore tech." apply_boem_osw,--,bool,"If true, overlay the BOEM OSW planning-area screen on the NREL exclusion raster for offwind and offwind_floating. Selects the ``_boem`` file variants. No effect on onshore tech." godeeep_wind_height,--,"str (e.g. ``_100m``, ``_125m``)","Hub-height suffix used to construct the raw GODEEEP wind capacity-factor filename (e.g. ``wind_gen_cf_2030_125m_compressed.nc``). Must match a published Zenodo record (``wind___compressed``). Ignored for solar." diff --git a/docs/source/release-notes.md b/docs/source/release-notes.md index b46dc9237..42d506890 100644 --- a/docs/source/release-notes.md +++ b/docs/source/release-notes.md @@ -49,6 +49,12 @@ changes you will notice: default several rules later. - **The EIA API key can be supplied as `$EIA_API_KEY`**, which takes precedence over `config/config.api.yaml` and keeps the key out of your files entirely. +- **`renewable_land_access` now defaults to `reference`.** The shipped templates previously + paired `renewable.dataset: godeeep` with a null land-access setting, a combination + `build_renewable_profiles` rejects — no shipped config could build renewable profiles from a + clean checkout. The default is NREL's central land-access assumption; expect + `retrieve_nrel_exclusion_artifact` jobs in every godeeep DAG. Null remains valid only with + `dataset: atlite`. ### Correctness fixes validated by an equivalence harness From fe8873ca0c3306e55f3e8e46c6bee638da0b08e8 Mon Sep 17 00:00:00 2001 From: ktehranchi Date: Sat, 29 Aug 2026 16:01:41 -0700 Subject: [PATCH 13/14] CI: install GLPK in the fast-tests job The repo-root conftest auto-marks every workflow/scripts/test/ test as 'fast', and those tests solve with GLPK - a system binary the pip-only fast-tests job never had. All 13 Tier A CI failures on re-enabling the workflow were 'Solver glpk not installed' (or its optimize-status variant); they pass locally where glpsol exists. Co-Authored-By: Claude Fable 5 --- .github/workflows/main.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 107a3e16f..fb2d4733c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -29,6 +29,8 @@ jobs: - uses: actions/setup-python@v5 with: python-version: '3.11' + - name: Install GLPK (solver for the auto-marked unit tests) + run: sudo apt-get update -qq && sudo apt-get install -y -qq glpk-utils - name: Install test extras run: pip install -e '.[test]' - name: Run Tier A (static checks) From 7ef1fc6b45ac44f9f6098060885eb5b95a584e8c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 31 Aug 2026 23:04:52 +0000 Subject: [PATCH 14/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- workflow/scripts/godeeep_cf_registry.py | 2 +- workflow/scripts/nrel_exclusion/make_compress_tasks.py | 9 +++++---- workflow/scripts/nrel_exclusion/stage_godeeep_member.py | 4 ++-- workflow/scripts/zenodo_downloader.py | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/workflow/scripts/godeeep_cf_registry.py b/workflow/scripts/godeeep_cf_registry.py index 1bf6d4e0e..ff11012b7 100644 --- a/workflow/scripts/godeeep_cf_registry.py +++ b/workflow/scripts/godeeep_cf_registry.py @@ -481,7 +481,7 @@ def _availability_report(sources: Sequence[CfSource], key: str) -> str: def _format_years(years: Sequence[int]) -> str: """Render a year tuple compactly, collapsing contiguous runs to a range.""" - ordered = sorted(set(int(y) for y in years)) + ordered = sorted({int(y) for y in years}) if not ordered: return "(none)" if len(ordered) > 2 and ordered == list(range(ordered[0], ordered[-1] + 1)): diff --git a/workflow/scripts/nrel_exclusion/make_compress_tasks.py b/workflow/scripts/nrel_exclusion/make_compress_tasks.py index 26f4833fc..9dbcf2f8d 100644 --- a/workflow/scripts/nrel_exclusion/make_compress_tasks.py +++ b/workflow/scripts/nrel_exclusion/make_compress_tasks.py @@ -96,7 +96,7 @@ def scan_tech(zip_dir: Path, tech: str, years: range) -> dict[int, tuple[Path, s prior_zip, prior_member, _ = found[year] raise ValueError( f"{tech} {year}: duplicate members " - f"{prior_zip.name}:{prior_member} and {archive.name}:{info.filename}" + f"{prior_zip.name}:{prior_member} and {archive.name}:{info.filename}", ) found[year] = (archive, info.filename, info.file_size) @@ -105,7 +105,7 @@ def scan_tech(zip_dir: Path, tech: str, years: range) -> dict[int, tuple[Path, s raise ValueError( f"{tech}: incomplete coverage — missing years {missing} " f"(searched {[a.name for a in archives]} in {zip_dir}); " - f"found {len(found)}/{len(wanted)} of {min(years)}-{max(years)}" + f"found {len(found)}/{len(wanted)} of {min(years)}-{max(years)}", ) return found @@ -130,7 +130,7 @@ def enumerate_tasks( member=member, member_bytes=size, dest_filename=dest_filename(member), - ) + ), ) return tasks @@ -166,7 +166,8 @@ def main() -> None: for tech in args.techs: rows = [task for task in tasks if task.tech == tech] print( - f"[tasks] {tech}: {len(rows)} years {min(t.year for t in rows)}-{max(t.year for t in rows)}", flush=True + f"[tasks] {tech}: {len(rows)} years {min(t.year for t in rows)}-{max(t.year for t in rows)}", + flush=True, ) print(f"[tasks] submit with --array=1-{len(tasks)}", flush=True) diff --git a/workflow/scripts/nrel_exclusion/stage_godeeep_member.py b/workflow/scripts/nrel_exclusion/stage_godeeep_member.py index 61bbfbfcb..66dc352d4 100644 --- a/workflow/scripts/nrel_exclusion/stage_godeeep_member.py +++ b/workflow/scripts/nrel_exclusion/stage_godeeep_member.py @@ -50,7 +50,7 @@ def stage_member( raise KeyError(f"member {member!r} not found in {zip_path}") from None if expect_bytes is not None and info.file_size != expect_bytes: raise ValueError( - f"{zip_path}:{member} central directory reports {info.file_size} bytes, task list expects {expect_bytes}" + f"{zip_path}:{member} central directory reports {info.file_size} bytes, task list expects {expect_bytes}", ) with zf.open(info) as src, open(dest, "wb") as out: shutil.copyfileobj(src, out, length=COPY_BLOCK) @@ -58,7 +58,7 @@ def stage_member( staged = dest.stat().st_size if staged != info.file_size: raise OSError( - f"staged {staged} bytes to {dest}, central directory says {info.file_size} for {zip_path}:{member}" + f"staged {staged} bytes to {dest}, central directory says {info.file_size} for {zip_path}:{member}", ) return staged diff --git a/workflow/scripts/zenodo_downloader.py b/workflow/scripts/zenodo_downloader.py index b9df6d08c..b2245f720 100644 --- a/workflow/scripts/zenodo_downloader.py +++ b/workflow/scripts/zenodo_downloader.py @@ -266,6 +266,6 @@ def _retry_or_raise(exc, attempt, what): ) from exc delay = BACKOFF_BASE_SECONDS * 2 ** (attempt - 1) print( - f"Transient error fetching {what} (attempt {attempt}/{MAX_DOWNLOAD_ATTEMPTS}): {exc}. Retrying in {delay:.0f}s..." + f"Transient error fetching {what} (attempt {attempt}/{MAX_DOWNLOAD_ATTEMPTS}): {exc}. Retrying in {delay:.0f}s...", ) time.sleep(delay)