Attach sector demand after spatial clustering

workflow/config/config.common.yaml

@@ -1,3 +1,7 @@
+pudl_cache:
+  version: v2025.5.0
+  base_url: https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop
+  directory: repo_data/pudl
 pudl_path: s3://pudl.catalyst.coop/v2025.2.0
 
 foresight: 'perfect'

workflow/repo_data/config/config.common.yaml

@@ -1,5 +1,10 @@
 pudl_path: s3://pudl.catalyst.coop/v2025.5.0
 
+pudl_cache:
+  version: v2025.5.0
+  base_url: https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop
+  directory: repo_data/pudl
+
 # docs :
 renewable:
   EGS:

workflow/rules/build_electricity.smk

@@ -1,6 +1,107 @@
 ################# ----------- Rules to Build Network ---------- #################
 
 from itertools import chain
+from pathlib import Path
+
+
+PUDL_VERSION = config["pudl_cache"]["version"]
+PUDL_BASE_URL = config["pudl_cache"]["base_url"].rstrip("/")
+PUDL_DIRECTORY = Path(config["pudl_cache"]["directory"]) / PUDL_VERSION
+
+PUDL_POWERPLANT_FILES = [
+    "out_eia__monthly_generators.parquet",
+    "out_eia__yearly_generators.parquet",
+    "core_eia860__scd_generators_energy_storage.parquet",
+    "core_eia860__scd_plants.parquet",
+]
+
+PUDL_COST_FILES = [
+    "core_nrelatb__yearly_projected_financial_cases_by_scenario.parquet",
+    "core_nrelatb__yearly_projected_cost_performance.parquet",
+    "core_eiaaeo__yearly_projected_fuel_cost_in_electric_sector_by_type.parquet",
+]
+
+PUDL_DEMAND_FILES = [
+    "censusdp1tract.state_2010census_dp1.parquet",
+    "core_eiaaeo__yearly_projected_generation_in_electric_sector_by_technology.parquet",
+]
+
+PUDL_FUEL_PRICE_FILES = [
+    "out_eia__monthly_generators.parquet",
+    "out_eia__yearly_generators.parquet",
+    "core_eia860__scd_plants.parquet",
+]
+
+PUDL_FILES = sorted(
+    set(
+        PUDL_POWERPLANT_FILES
+        + PUDL_COST_FILES
+        + PUDL_DEMAND_FILES
+        + PUDL_FUEL_PRICE_FILES
+    )
+)
+
+
+def pudl_paths(filenames):
+    return [str(PUDL_DIRECTORY / filename) for filename in filenames]
+
+
+PUDL_POWERPLANT_PATHS = pudl_paths(PUDL_POWERPLANT_FILES)
+PUDL_COST_PATHS = pudl_paths(PUDL_COST_FILES)
+PUDL_FUEL_PRICE_PATHS = pudl_paths(PUDL_FUEL_PRICE_FILES)
+
+
+def demand_pudl_paths(wildcards):
+    profile = config["electricity"]["demand"]["profile"]
+
+    if profile == "ferc":
+        return pudl_paths(PUDL_DEMAND_FILES)
+
+    if profile == "eia":
+        return pudl_paths(
+            [
+                "core_eiaaeo__yearly_projected_generation_in_electric_sector_by_technology.parquet",
+            ]
+        )
+
+    return []
+
+
+rule retrieve_pudl_file:
+    output:
+        str(PUDL_DIRECTORY / "{pudl_file}.parquet"),
+    params:
+        url=lambda wildcards: (
+            f"{PUDL_BASE_URL}/{PUDL_VERSION}/" f"{wildcards.pudl_file}.parquet"
+        ),
+    wildcard_constraints:
+        pudl_file="|".join(Path(filename).stem for filename in PUDL_FILES),
+    log:
+        "logs/retrieve_pudl/{pudl_file}.log",
+    resources:
+        mem_mb=1000,
+        walltime="00:30:00",
+    shell:
+        r"""
+        mkdir -p "$(dirname {output:q})" "$(dirname {log:q})"
+
+        curl -fL \
+          --retry 5 \
+          --retry-all-errors \
+          --connect-timeout 20 \
+          --max-time 1800 \
+          {params.url:q} \
+          --output {output:q}.part \
+          > {log:q} 2>&1
+
+        test -s {output:q}.part
+
+        python -c \
+          "import duckdb; duckdb.sql(\"SELECT * FROM read_parquet('{output}.part') LIMIT 1\").fetchall()" \
+          >> {log:q} 2>&1
+
+        mv {output:q}.part {output:q}
+        """
 
 
 rule build_shapes:
@@ -101,8 +202,9 @@ rule build_bus_regions:
 rule build_cost_data:
     params:
         aeo=config_provider("costs", "aeo"),
-        pudl_path=config_provider("pudl_path"),
+        pudl_path=str(PUDL_DIRECTORY),
     input:
+        pudl=PUDL_COST_PATHS,
         efs_tech_costs="repo_data/costs/EFS_Technology_Data.xlsx",
         efs_icev_costs="repo_data/costs/efs_icev_costs.csv",
         eia_tech_costs="repo_data/costs/eia_tech_costs.csv",
@@ -344,7 +446,6 @@ def demand_raw_data(wildcards):
     elif profile == "ferc":
         return [
             DATA + "pudl/out_ferc714__hourly_estimated_state_demand.parquet",
-            DATA + "pudl/censusdp1tract.sqlite",
         ]
     elif profile == "eulp":
         return [
@@ -422,8 +523,9 @@ rule build_electrical_demand:
         planning_horizons=config["scenario"]["planning_horizons"],
         renewable_weather_years=config["renewable_weather_years"],
         snapshots=config["snapshots"],
-        pudl_path=config_provider("pudl_path"),
+        pudl_path=str(PUDL_DIRECTORY),
     input:
+        pudl=demand_pudl_paths,
         network=RESOURCES + "{interconnect}/elec_base_network.nc",
         demand_files=demand_raw_data,
         demand_scaling_file=demand_scaling_data,
@@ -457,10 +559,12 @@ rule build_service_demand:
         dissagregate_files=demand_dissagregate_data,
         demand_scaling_file=demand_scaling_data,
     output:
-        electricity=RESOURCES + "{interconnect}/demand/{end_use}_electricity.pkl",
-        space_heat=RESOURCES + "{interconnect}/demand/{end_use}_space-heating.pkl",
-        water_heat=RESOURCES + "{interconnect}/demand/{end_use}_water-heating.pkl",
-        cool=RESOURCES + "{interconnect}/demand/{end_use}_cooling.pkl",
+        electricity=RESOURCES + "{interconnect}/demand/sector/{end_use}_electricity.pkl",
+        space_heat=RESOURCES
+        + "{interconnect}/demand/sector/{end_use}_space-heating.pkl",
+        water_heat=RESOURCES
+        + "{interconnect}/demand/sector/{end_use}_water-heating.pkl",
+        cool=RESOURCES + "{interconnect}/demand/sector/{end_use}_cooling.pkl",
     log:
         LOGS + "{interconnect}/demand/{end_use}_build_demand.log",
     benchmark:
@@ -480,15 +584,14 @@ rule build_industry_demand:
         profile_year=pd.to_datetime(config["snapshots"]["start"]).year,
         eia_api=config_provider("api", "eia"),
         snapshots=config_provider("snapshots"),
-        pudl_path=config_provider("pudl_path"),
     input:
         network=RESOURCES + "{interconnect}/elec_base_network.nc",
         demand_files=demand_raw_data,
         dissagregate_files=demand_dissagregate_data,
         demand_scaling_file=demand_scaling_data,
     output:
-        electricity=RESOURCES + "{interconnect}/demand/{end_use}_electricity.pkl",
-        heat=RESOURCES + "{interconnect}/demand/{end_use}_heating.pkl",
+        electricity=RESOURCES + "{interconnect}/demand/sector/{end_use}_electricity.pkl",
+        heat=RESOURCES + "{interconnect}/demand/sector/{end_use}_heating.pkl",
     log:
         LOGS + "{interconnect}/demand/{end_use}_build_demand.log",
     benchmark:
@@ -515,10 +618,10 @@ rule build_transport_road_demand:
         dissagregate_files=demand_dissagregate_data,
         demand_scaling_file=demand_scaling_data,
     output:
-        light_duty=RESOURCES + "{interconnect}/demand/{end_use}_light-duty.pkl",
-        med_duty=RESOURCES + "{interconnect}/demand/{end_use}_med-duty.pkl",
-        heavy_duty=RESOURCES + "{interconnect}/demand/{end_use}_heavy-duty.pkl",
-        bus=RESOURCES + "{interconnect}/demand/{end_use}_bus.pkl",
+        light_duty=RESOURCES + "{interconnect}/demand/sector/{end_use}_light-duty.pkl",
+        med_duty=RESOURCES + "{interconnect}/demand/sector/{end_use}_med-duty.pkl",
+        heavy_duty=RESOURCES + "{interconnect}/demand/sector/{end_use}_heavy-duty.pkl",
+        bus=RESOURCES + "{interconnect}/demand/sector/{end_use}_bus.pkl",
     log:
         LOGS + "{interconnect}/demand/{end_use}_build_demand.log",
     benchmark:
@@ -546,7 +649,7 @@ rule build_transport_other_demand:
         demand_files=demand_raw_data,
         dissagregate_files=demand_dissagregate_data,
     output:
-        RESOURCES + "{interconnect}/demand/{end_use}_{vehicle}.pkl",
+        RESOURCES + "{interconnect}/demand/sector/{end_use}_{vehicle}.pkl",
     log:
         LOGS + "{interconnect}/demand/{end_use}_{vehicle}_build_demand.log",
     benchmark:
@@ -558,53 +661,80 @@ rule build_transport_other_demand:
         "../scripts/build_demand.py"
 
 
-def demand_to_add(wildcards):
+def sector_demand_files(wildcards):
+    """
+    Return compact sector-demand inputs for the clustered network.
+
+    Parameters
+    ----------
+    wildcards : snakemake.io.Wildcards
+        Wildcards supplied by Snakemake. Returned paths retain the existing
+        interconnect placeholder for workflow expansion.
+
+    Returns
+    -------
+    list or itertools.chain
+        An empty list for electricity-only studies. For sector studies,
+        an iterator over residential, commercial, industrial, road-transport,
+        and other transport demand files.
+
+    Notes
+    -----
+    These files are consumed by add_extra_components after spatial
+    clustering. They are not inputs to the initial add_demand rule.
+
+    The existing service-sector configuration determines whether heating
+    demand is split into space and water heating. No new configuration
+    options are introduced.
+    """
+    if config["scenario"]["sector"] in ("E", ""):
+        return []
 
-    if config["scenario"]["sector"] == "E":
-        return RESOURCES + "{interconnect}/demand/power_electricity.csv"
+    services = ["residential", "commercial"]
+    if config["sector"]["service_sector"]["split_space_water_heating"]:
+        fuels = ["electricity", "cooling", "space-heating", "water-heating"]
     else:
-        # service demand
-        services = ["residential", "commercial"]
-        if config["sector"]["service_sector"]["split_space_water_heating"]:
-            fuels = ["electricity", "cooling", "space-heating", "water-heating"]
-        else:
-            fuels = ["electricity", "cooling", "heating"]
-        service_demands = [
-            RESOURCES + "{interconnect}/demand/" + service + "_" + fuel + ".pkl"
-            for service in services
-            for fuel in fuels
-        ]
-        # industrial demand
-        fuels = ["electricity", "heating"]
-        industrial_demands = [
-            RESOURCES + "{interconnect}/demand/industry_" + fuel + ".pkl"
-            for fuel in fuels
-        ]
-        # road transport demands
-        vehicles = ["light-duty", "med-duty", "heavy-duty", "bus"]
-        road_demand = [
-            RESOURCES + "{interconnect}/demand/transport_" + vehicle + ".pkl"
-            for vehicle in vehicles
-        ]
-
-        # other transport demands
-        vehicles = ["boat-shipping", "rail-shipping", "rail-passenger", "air"]
-        non_road_demand = [
-            RESOURCES + "{interconnect}/demand/transport_" + vehicle + ".pkl"
-            for vehicle in vehicles
-        ]
-
-        return chain(service_demands, industrial_demands, road_demand, non_road_demand)
+        fuels = ["electricity", "cooling", "heating"]
+
+    service_demands = [
+        RESOURCES + "{interconnect}/demand/sector/" + service + "_" + fuel + ".pkl"
+        for service in services
+        for fuel in fuels
+    ]
+
+    fuels = ["electricity", "heating"]
+    industrial_demands = [
+        RESOURCES + "{interconnect}/demand/sector/industry_" + fuel + ".pkl"
+        for fuel in fuels
+    ]
+
+    vehicles = ["light-duty", "med-duty", "heavy-duty", "bus"]
+    road_demand = [
+        RESOURCES + "{interconnect}/demand/sector/transport_" + vehicle + ".pkl"
+        for vehicle in vehicles
+    ]
+
+    vehicles = ["boat-shipping", "rail-shipping", "rail-passenger", "air"]
+    non_road_demand = [
+        RESOURCES + "{interconnect}/demand/sector/transport_" + vehicle + ".pkl"
+        for vehicle in vehicles
+    ]
+
+    return chain(
+        service_demands,
+        industrial_demands,
+        road_demand,
+        non_road_demand,
+    )
 
 
 rule add_demand:
     params:
-        sectors=config["scenario"]["sector"],
         planning_horizons=config_provider("scenario", "planning_horizons"),
         snapshots=config_provider("snapshots"),
     input:
         network=RESOURCES + "{interconnect}/elec_base_network.nc",
-        demand=demand_to_add,
+        demand=RESOURCES + "{interconnect}/demand/power_electricity.csv",
     output:
         network=RESOURCES + "{interconnect}/elec_base_network_dem.nc",
     log:
@@ -629,8 +759,9 @@ rule build_fuel_prices:
     params:
         snapshots=config["snapshots"],
         api_eia=config["api"]["eia"],
-        pudl_path=config_provider("pudl_path"),
+        pudl_path=str(PUDL_DIRECTORY),
     input:
+        pudl=PUDL_FUEL_PRICE_PATHS,
         gas_balancing_area=ba_gas_dynamic_fuel_price_files,
     output:
         state_ng_fuel_prices=RESOURCES + "{interconnect}/state_ng_power_prices.csv",
@@ -665,8 +796,10 @@ def dynamic_fuel_price_files(wildcards):
 
 rule build_powerplants:
     params:
-        pudl_path=config_provider("pudl_path"),
+        pudl_path=str(PUDL_DIRECTORY),
+        renewable_weather_year=config_provider("renewable_weather_years"),
     input:
+        pudl=PUDL_POWERPLANT_PATHS,
         wecc_ads="repo_data/WECC_ADS_public",
         eia_ads_generator_mapping="repo_data/WECC_ADS_public/eia_ads_generator_mapping_updated.csv",
         fuel_costs="repo_data/plants/fuelCost22.csv",
@@ -790,6 +923,7 @@ rule simplify_network:
         + "{interconnect}/Geospatial/regions_offshore.geojson",
     output:
         network=RESOURCES + "{interconnect}/elec_s{simpl}.nc",
+        busmap=RESOURCES + "{interconnect}/busmap_s{simpl}.csv",
         regions_onshore=RESOURCES
         + "{interconnect}/Geospatial/regions_onshore_s{simpl}.geojson",
         regions_offshore=RESOURCES
@@ -881,6 +1015,17 @@ rule add_extra_components:
             if hour.isdigit()
         },
         network=RESOURCES + "{interconnect}/elec_s{simpl}_c{clusters}.nc",
+        sector_demand=lambda w: list(sector_demand_files(w)),
+        busmap_s=(
+            RESOURCES + "{interconnect}/busmap_s{simpl}.csv"
+            if config["scenario"]["sector"] not in ("E", "")
+            else []
+        ),
+        busmap_c=(
+            RESOURCES + "{interconnect}/busmap_s{simpl}_{clusters}.csv"
+            if config["scenario"]["sector"] not in ("E", "")
+            else []
+        ),
         tech_costs=lambda wildcards: expand(
             RESOURCES + "costs/costs_{year}.csv",
             year=config["scenario"]["planning_horizons"],

workflow/rules/common.smk

@@ -7,8 +7,11 @@ from functools import partial, lru_cache
 
 import os, sys, glob
 
-path = workflow.source_path("../scripts/_helpers.py")
-sys.path.insert(0, os.path.dirname(path))
+# Cache local modules before importing them from Snakemake's source cache.
+for source in ("../scripts/_helpers.py", "../scripts/constants.py"):
+    source_dir = os.path.dirname(workflow.source_path(source))
+    if source_dir not in sys.path:
+        sys.path.insert(0, source_dir)
 
 from _helpers import validate_checksum, update_config_from_wildcards
 from constants import HOURS_PER_YEAR