From 3658827d201248e9b93c1b81b3b5fd0e948d2b97 Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Thu, 6 Aug 2026 22:20:57 +0200 Subject: [PATCH 01/13] Implement workaround for PUDL files download --- workflow/scripts/build_powerplants.py | 78 ++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 6 deletions(-) diff --git a/workflow/scripts/build_powerplants.py b/workflow/scripts/build_powerplants.py index 29c509554..c91d110e9 100644 --- a/workflow/scripts/build_powerplants.py +++ b/workflow/scripts/build_powerplants.py @@ -1,7 +1,11 @@ """Assimilates data on existing generator and storage resources from PUDL, CEMS, ADS, and other sources.""" import logging +import os import re +import subprocess +from pathlib import Path +from urllib.parse import urlparse import constants as const import duckdb @@ -12,9 +16,71 @@ logger = logging.getLogger(__name__) -def initialize_duckdb(): - duckdb.connect(database=":memory:", read_only=False) - duckdb.query("INSTALL httpfs;") +PUDL_PARQUET_FILES = ( + "out_eia__monthly_generators.parquet", + "out_eia__yearly_generators.parquet", + "core_eia860__scd_generators_energy_storage.parquet", + "core_eia860__scd_plants.parquet", +) + + +def _pudl_https_url(pudl_path: str) -> str: + """Return an HTTPS base URL that has a valid TLS certificate.""" + path = pudl_path.rstrip("/") + s3_prefix = "s3://pudl.catalyst.coop" + virtual_hosted_prefix = "https://pudl.catalyst.coop.s3.amazonaws.com" + path_style_prefix = "https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop" + + if path.startswith(s3_prefix): + return path.replace(s3_prefix, path_style_prefix, 1) + if path.startswith(virtual_hosted_prefix): + return path.replace(virtual_hosted_prefix, path_style_prefix, 1) + return path + + +def cache_pudl_parquet_files(pudl_path: str) -> str: + """Download required remote PUDL Parquet files and return a local path.""" + parsed = urlparse(pudl_path) + if parsed.scheme not in {"s3", "http", "https"}: + return pudl_path + + base_url = _pudl_https_url(pudl_path) + version = Path(parsed.path).name or "current" + cache_root = Path(os.environ.get("PUDL_CACHE_DIR", "repo_data/pudl")) + cache_dir = cache_root / version + cache_dir.mkdir(parents=True, exist_ok=True) + + for filename in PUDL_PARQUET_FILES: + destination = cache_dir / filename + if destination.is_file() and destination.stat().st_size > 0: + logger.info("Using cached PUDL file %s", destination) + continue + + temporary = destination.with_suffix(f"{destination.suffix}.part") + logger.info("Downloading %s", filename) + try: + subprocess.run( + [ + "curl", + "-fL", + "--retry", + "3", + "--connect-timeout", + "10", + "--max-time", + "1800", + f"{base_url}/{filename}", + "-o", + str(temporary), + ], + check=True, + ) + temporary.replace(destination) + except (OSError, subprocess.CalledProcessError): + temporary.unlink(missing_ok=True) + raise + + return str(cache_dir) def load_eia_operable_data(parquet_path: str): @@ -694,9 +760,9 @@ def apply_cems_heat_rates(plants, crosswalk_fn, cems_fn): start_date = f"{data_year}-01-01" end_date = f"{data_year + 1}-01-01" - initialize_duckdb() - eia_data_operable = load_eia_operable_data(snakemake.params.pudl_path) - heat_rates = load_heat_rates_data(snakemake.params.pudl_path, start_date, end_date) + pudl_path = cache_pudl_parquet_files(snakemake.params.pudl_path) + eia_data_operable = load_eia_operable_data(pudl_path) + heat_rates = load_heat_rates_data(pudl_path, start_date, end_date) eia_data_operable = merge_fc_hr_data( eia_data_operable, From f0979a76de0abb5d075ba7e4769aac7fb000d76b Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Thu, 6 Aug 2026 22:43:23 +0200 Subject: [PATCH 02/13] Download PUDL files locally for power plant build --- workflow/repo_data/config/config.common.yaml | 5 ++ workflow/rules/build_electricity.smk | 52 +++++++++++++- workflow/scripts/build_powerplants.py | 73 +------------------- 3 files changed, 57 insertions(+), 73 deletions(-) diff --git a/workflow/repo_data/config/config.common.yaml b/workflow/repo_data/config/config.common.yaml index a5900e7eb..8577cf638 100644 --- a/workflow/repo_data/config/config.common.yaml +++ b/workflow/repo_data/config/config.common.yaml @@ -1,5 +1,10 @@ pudl_path: s3://pudl.catalyst.coop/v2025.5.0 +pudl_cache: + version: v2025.5.0 + base_url: https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop + directory: repo_data/pudl + # docs : renewable: EGS: diff --git a/workflow/rules/build_electricity.smk b/workflow/rules/build_electricity.smk index e45204d79..7c23912a0 100644 --- a/workflow/rules/build_electricity.smk +++ b/workflow/rules/build_electricity.smk @@ -1,6 +1,7 @@ ################# ----------- Rules to Build Network ---------- ################# from itertools import chain +from pathlib import Path rule build_shapes: @@ -663,10 +664,59 @@ def dynamic_fuel_price_files(wildcards): return {} +PUDL_POWERPLANT_FILES = [ + "out_eia__monthly_generators.parquet", + "out_eia__yearly_generators.parquet", + "core_eia860__scd_generators_energy_storage.parquet", + "core_eia860__scd_plants.parquet", +] + +PUDL_VERSION = config["pudl_cache"]["version"] +PUDL_BASE_URL = config["pudl_cache"]["base_url"].rstrip("/") +PUDL_DIRECTORY = Path(config["pudl_cache"]["directory"]) / PUDL_VERSION + +PUDL_POWERPLANT_PATHS = [ + str(PUDL_DIRECTORY / filename) for filename in PUDL_POWERPLANT_FILES +] + + +rule retrieve_pudl_powerplant_file: + output: + str(PUDL_DIRECTORY / "{pudl_file}.parquet"), + params: + url=lambda wildcards: ( + f"{PUDL_BASE_URL}/{PUDL_VERSION}/" f"{wildcards.pudl_file}.parquet" + ), + wildcard_constraints: + pudl_file="|".join(Path(filename).stem for filename in PUDL_POWERPLANT_FILES), + log: + "logs/retrieve_pudl/{pudl_file}.log", + resources: + mem_mb=1000, + walltime="00:30:00", + shell: + r""" + mkdir -p "$(dirname {output:q})" "$(dirname {log:q})" + + curl -fL \ + --retry 5 \ + --retry-all-errors \ + --connect-timeout 20 \ + --max-time 1800 \ + {params.url:q} \ + --output {output:q}.part \ + > {log:q} 2>&1 + + mv {output:q}.part {output:q} + """ + + rule build_powerplants: params: - pudl_path=config_provider("pudl_path"), + pudl_path=str(PUDL_DIRECTORY), + renewable_weather_year=config_provider("renewable_weather_years"), input: + pudl=PUDL_POWERPLANT_PATHS, wecc_ads="repo_data/WECC_ADS_public", eia_ads_generator_mapping="repo_data/WECC_ADS_public/eia_ads_generator_mapping_updated.csv", fuel_costs="repo_data/plants/fuelCost22.csv", diff --git a/workflow/scripts/build_powerplants.py b/workflow/scripts/build_powerplants.py index c91d110e9..aa64c46f7 100644 --- a/workflow/scripts/build_powerplants.py +++ b/workflow/scripts/build_powerplants.py @@ -1,11 +1,7 @@ """Assimilates data on existing generator and storage resources from PUDL, CEMS, ADS, and other sources.""" import logging -import os import re -import subprocess -from pathlib import Path -from urllib.parse import urlparse import constants as const import duckdb @@ -16,73 +12,6 @@ logger = logging.getLogger(__name__) -PUDL_PARQUET_FILES = ( - "out_eia__monthly_generators.parquet", - "out_eia__yearly_generators.parquet", - "core_eia860__scd_generators_energy_storage.parquet", - "core_eia860__scd_plants.parquet", -) - - -def _pudl_https_url(pudl_path: str) -> str: - """Return an HTTPS base URL that has a valid TLS certificate.""" - path = pudl_path.rstrip("/") - s3_prefix = "s3://pudl.catalyst.coop" - virtual_hosted_prefix = "https://pudl.catalyst.coop.s3.amazonaws.com" - path_style_prefix = "https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop" - - if path.startswith(s3_prefix): - return path.replace(s3_prefix, path_style_prefix, 1) - if path.startswith(virtual_hosted_prefix): - return path.replace(virtual_hosted_prefix, path_style_prefix, 1) - return path - - -def cache_pudl_parquet_files(pudl_path: str) -> str: - """Download required remote PUDL Parquet files and return a local path.""" - parsed = urlparse(pudl_path) - if parsed.scheme not in {"s3", "http", "https"}: - return pudl_path - - base_url = _pudl_https_url(pudl_path) - version = Path(parsed.path).name or "current" - cache_root = Path(os.environ.get("PUDL_CACHE_DIR", "repo_data/pudl")) - cache_dir = cache_root / version - cache_dir.mkdir(parents=True, exist_ok=True) - - for filename in PUDL_PARQUET_FILES: - destination = cache_dir / filename - if destination.is_file() and destination.stat().st_size > 0: - logger.info("Using cached PUDL file %s", destination) - continue - - temporary = destination.with_suffix(f"{destination.suffix}.part") - logger.info("Downloading %s", filename) - try: - subprocess.run( - [ - "curl", - "-fL", - "--retry", - "3", - "--connect-timeout", - "10", - "--max-time", - "1800", - f"{base_url}/{filename}", - "-o", - str(temporary), - ], - check=True, - ) - temporary.replace(destination) - except (OSError, subprocess.CalledProcessError): - temporary.unlink(missing_ok=True) - raise - - return str(cache_dir) - - def load_eia_operable_data(parquet_path: str): """Queries the parquet files directly for operable plant data.""" return duckdb.query( @@ -760,7 +689,7 @@ def apply_cems_heat_rates(plants, crosswalk_fn, cems_fn): start_date = f"{data_year}-01-01" end_date = f"{data_year + 1}-01-01" - pudl_path = cache_pudl_parquet_files(snakemake.params.pudl_path) + pudl_path = snakemake.params.pudl_path eia_data_operable = load_eia_operable_data(pudl_path) heat_rates = load_heat_rates_data(pudl_path, start_date, end_date) From 97f38a3e565d4f43f650099ffc5874a69bbabac3 Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Fri, 7 Aug 2026 10:45:38 +0200 Subject: [PATCH 03/13] Fix PUDL data access using explicit local cache inputs --- workflow/rules/build_electricity.smk | 159 ++++++++++++++++++--------- workflow/scripts/build_cost_data.py | 9 -- workflow/scripts/build_demand.py | 10 +- 3 files changed, 107 insertions(+), 71 deletions(-) diff --git a/workflow/rules/build_electricity.smk b/workflow/rules/build_electricity.smk index 7c23912a0..e20fa930d 100644 --- a/workflow/rules/build_electricity.smk +++ b/workflow/rules/build_electricity.smk @@ -4,6 +4,106 @@ from itertools import chain from pathlib import Path +PUDL_VERSION = config["pudl_cache"]["version"] +PUDL_BASE_URL = config["pudl_cache"]["base_url"].rstrip("/") +PUDL_DIRECTORY = Path(config["pudl_cache"]["directory"]) / PUDL_VERSION + +PUDL_POWERPLANT_FILES = [ + "out_eia__monthly_generators.parquet", + "out_eia__yearly_generators.parquet", + "core_eia860__scd_generators_energy_storage.parquet", + "core_eia860__scd_plants.parquet", +] + +PUDL_COST_FILES = [ + "core_nrelatb__yearly_projected_financial_cases_by_scenario.parquet", + "core_nrelatb__yearly_projected_cost_performance.parquet", + "core_eiaaeo__yearly_projected_fuel_cost_in_electric_sector_by_type.parquet", +] + +PUDL_DEMAND_FILES = [ + "censusdp1tract.state_2010census_dp1.parquet", + "core_eiaaeo__yearly_projected_generation_in_electric_sector_by_technology.parquet", +] + +PUDL_FUEL_PRICE_FILES = [ + "out_eia__monthly_generators.parquet", + "out_eia__yearly_generators.parquet", + "core_eia860__scd_plants.parquet", +] + +PUDL_FILES = sorted( + set( + PUDL_POWERPLANT_FILES + + PUDL_COST_FILES + + PUDL_DEMAND_FILES + + PUDL_FUEL_PRICE_FILES + ) +) + + +def pudl_paths(filenames): + return [str(PUDL_DIRECTORY / filename) for filename in filenames] + + +PUDL_POWERPLANT_PATHS = pudl_paths(PUDL_POWERPLANT_FILES) +PUDL_COST_PATHS = pudl_paths(PUDL_COST_FILES) +PUDL_FUEL_PRICE_PATHS = pudl_paths(PUDL_FUEL_PRICE_FILES) + + +def demand_pudl_paths(wildcards): + profile = config["electricity"]["demand"]["profile"] + + if profile == "ferc": + return pudl_paths(PUDL_DEMAND_FILES) + + if profile == "eia": + return pudl_paths( + [ + "core_eiaaeo__yearly_projected_generation_in_electric_sector_by_technology.parquet", + ] + ) + + return [] + + +rule retrieve_pudl_file: + output: + str(PUDL_DIRECTORY / "{pudl_file}.parquet"), + params: + url=lambda wildcards: ( + f"{PUDL_BASE_URL}/{PUDL_VERSION}/" f"{wildcards.pudl_file}.parquet" + ), + wildcard_constraints: + pudl_file="|".join(Path(filename).stem for filename in PUDL_FILES), + log: + "logs/retrieve_pudl/{pudl_file}.log", + resources: + mem_mb=1000, + walltime="00:30:00", + shell: + r""" + mkdir -p "$(dirname {output:q})" "$(dirname {log:q})" + + curl -fL \ + --retry 5 \ + --retry-all-errors \ + --connect-timeout 20 \ + --max-time 1800 \ + {params.url:q} \ + --output {output:q}.part \ + > {log:q} 2>&1 + + test -s {output:q}.part + + python -c \ + "import duckdb; duckdb.sql(\"SELECT * FROM read_parquet('{output}.part') LIMIT 1\").fetchall()" \ + >> {log:q} 2>&1 + + mv {output:q}.part {output:q} + """ + + rule build_shapes: params: source_offshore_shapes=config_provider("offshore_shape"), @@ -102,8 +202,9 @@ rule build_bus_regions: rule build_cost_data: params: aeo=config_provider("costs", "aeo"), - pudl_path=config_provider("pudl_path"), + pudl_path=str(PUDL_DIRECTORY), input: + pudl=PUDL_COST_PATHS, efs_tech_costs="repo_data/costs/EFS_Technology_Data.xlsx", efs_icev_costs="repo_data/costs/efs_icev_costs.csv", eia_tech_costs="repo_data/costs/eia_tech_costs.csv", @@ -121,7 +222,6 @@ rule build_cost_data: script: "../scripts/build_cost_data.py" - ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4) if config["enable"].get("build_cutout", False): @@ -345,7 +445,6 @@ def demand_raw_data(wildcards): elif profile == "ferc": return [ DATA + "pudl/out_ferc714__hourly_estimated_state_demand.parquet", - DATA + "pudl/censusdp1tract.sqlite", ] elif profile == "eulp": return [ @@ -423,8 +522,9 @@ rule build_electrical_demand: planning_horizons=config["scenario"]["planning_horizons"], renewable_weather_years=config["renewable_weather_years"], snapshots=config["snapshots"], - pudl_path=config_provider("pudl_path"), + pudl_path=str(PUDL_DIRECTORY), input: + pudl=demand_pudl_paths, network=RESOURCES + "{interconnect}/elec_base_network.nc", demand_files=demand_raw_data, demand_scaling_file=demand_scaling_data, @@ -481,7 +581,6 @@ rule build_industry_demand: profile_year=pd.to_datetime(config["snapshots"]["start"]).year, eia_api=config_provider("api", "eia"), snapshots=config_provider("snapshots"), - pudl_path=config_provider("pudl_path"), input: network=RESOURCES + "{interconnect}/elec_base_network.nc", demand_files=demand_raw_data, @@ -630,8 +729,9 @@ rule build_fuel_prices: params: snapshots=config["snapshots"], api_eia=config["api"]["eia"], - pudl_path=config_provider("pudl_path"), + pudl_path=str(PUDL_DIRECTORY), input: + pudl=PUDL_FUEL_PRICE_PATHS, gas_balancing_area=ba_gas_dynamic_fuel_price_files, output: state_ng_fuel_prices=RESOURCES + "{interconnect}/state_ng_power_prices.csv", @@ -664,53 +764,6 @@ def dynamic_fuel_price_files(wildcards): return {} -PUDL_POWERPLANT_FILES = [ - "out_eia__monthly_generators.parquet", - "out_eia__yearly_generators.parquet", - "core_eia860__scd_generators_energy_storage.parquet", - "core_eia860__scd_plants.parquet", -] - -PUDL_VERSION = config["pudl_cache"]["version"] -PUDL_BASE_URL = config["pudl_cache"]["base_url"].rstrip("/") -PUDL_DIRECTORY = Path(config["pudl_cache"]["directory"]) / PUDL_VERSION - -PUDL_POWERPLANT_PATHS = [ - str(PUDL_DIRECTORY / filename) for filename in PUDL_POWERPLANT_FILES -] - - -rule retrieve_pudl_powerplant_file: - output: - str(PUDL_DIRECTORY / "{pudl_file}.parquet"), - params: - url=lambda wildcards: ( - f"{PUDL_BASE_URL}/{PUDL_VERSION}/" f"{wildcards.pudl_file}.parquet" - ), - wildcard_constraints: - pudl_file="|".join(Path(filename).stem for filename in PUDL_POWERPLANT_FILES), - log: - "logs/retrieve_pudl/{pudl_file}.log", - resources: - mem_mb=1000, - walltime="00:30:00", - shell: - r""" - mkdir -p "$(dirname {output:q})" "$(dirname {log:q})" - - curl -fL \ - --retry 5 \ - --retry-all-errors \ - --connect-timeout 20 \ - --max-time 1800 \ - {params.url:q} \ - --output {output:q}.part \ - > {log:q} 2>&1 - - mv {output:q}.part {output:q} - """ - - rule build_powerplants: params: pudl_path=str(PUDL_DIRECTORY), diff --git a/workflow/scripts/build_cost_data.py b/workflow/scripts/build_cost_data.py index 43a865df9..14a6a8ef2 100644 --- a/workflow/scripts/build_cost_data.py +++ b/workflow/scripts/build_cost_data.py @@ -63,17 +63,8 @@ ] # https://github.com/NREL/ReEDS-2.0/blob/e65ed5ed4ffff973071839481309f77d12d802cd/inputs/plant_characteristics/maxage.csv#L4 -def create_duckdb_instance(): - """Set up DuckDB to read parquet files directly.""" - duckdb.connect(database=":memory:", read_only=False) - # Install httpfs extension to access remote files if needed - duckdb.query("INSTALL httpfs;") - - def load_pudl_atb_data(parquet_path: str): """Loads ATB data directly from parquet files.""" - create_duckdb_instance() - query = f""" WITH finance_cte AS ( SELECT diff --git a/workflow/scripts/build_demand.py b/workflow/scripts/build_demand.py index 035c8766b..d7cebaed1 100644 --- a/workflow/scripts/build_demand.py +++ b/workflow/scripts/build_demand.py @@ -255,9 +255,6 @@ def _read_data(self) -> pd.DataFrame: def _read_census_data(self) -> pd.DataFrame: """Reads in census data for population weighting using parquet.""" - duckdb.connect(database=":memory:", read_only=False) - duckdb.query("INSTALL httpfs;") - parquet_path = snakemake.params.pudl_path sql = f""" @@ -2022,9 +2019,7 @@ def assign_scaler(self): # type DemandScaler assert self.api, "Must provide eia api key" return AeoEnergyScaler(self.api) elif self.scaling_method == "aeo_electricity": - assert self.filepath.startswith( - "s3://pudl.catalyst.coop/", - ), "Must provide pudl S3 URL (s3://pudl.catalyst.coop/...)" + assert self.filepath, "Must provide a local PUDL directory" return AeoElectricityScaler(self.filepath) elif self.scaling_method == "efs": assert self.filepath.endswith(".csv"), "Must provide EFS.csv data" @@ -2126,9 +2121,6 @@ def get_projections(self) -> pd.DataFrame: | 2049 | ### | ### | | 2050 | ### | ### | """ - duckdb.connect(database=":memory:", read_only=False) - duckdb.query("INSTALL httpfs;") - query = f""" SELECT projection_year, From fa8ae8d916f98007bf1ab3dcb12774d639635d1e Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Fri, 7 Aug 2026 11:19:37 +0200 Subject: [PATCH 04/13] Update NLR Data Catalog URLs --- workflow/scripts/build_demand.py | 4 ++-- workflow/scripts/build_sector_costs.py | 18 +++++++++--------- workflow/scripts/build_stock_data.py | 2 +- workflow/scripts/opts/sector.py | 2 +- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/workflow/scripts/build_demand.py b/workflow/scripts/build_demand.py index d7cebaed1..10f23d979 100644 --- a/workflow/scripts/build_demand.py +++ b/workflow/scripts/build_demand.py @@ -696,7 +696,7 @@ class ReadCliu(ReadStrategy): - MECS is used to scale county level data Sources: - - https://data.nrel.gov/submissions/97 + - https://data.nlr.gov/submissions/97 - https://www.eia.gov/consumption/manufacturing/data/2014/#r3 (Table 3.2) - https://loadshape.epri.com/enduse - https://github.com/NREL/Industry-Energy-Tool/ @@ -1811,7 +1811,7 @@ class WriteIndustrial(WriteStrategy): """ Based on county level energy use from 2014. - https://data.nrel.gov/submissions/97 + https://data.nlr.gov/submissions/97 """ def __init__(self, n: pypsa.Network, filepath: str) -> None: diff --git a/workflow/scripts/build_sector_costs.py b/workflow/scripts/build_sector_costs.py index 083931ca2..264f141cc 100644 --- a/workflow/scripts/build_sector_costs.py +++ b/workflow/scripts/build_sector_costs.py @@ -2,7 +2,7 @@ Builds costs data from EFS studies. https://www.nrel.gov/docs/fy18osti/70485.pdf -https://data.nrel.gov/submissions/78 +https://data.nlr.gov/submissions/78 """ from abc import ABC, abstractmethod @@ -22,7 +22,7 @@ # fixed maintenance costs in %/year # From https://www.nrel.gov/docs/fy18osti/71500.pdf -# Data in Sheet "13" at https://data.nrel.gov/submissions/93 +# Data in Sheet "13" at https://data.nlr.gov/submissions/93 # manually claculated as fixed/capex ICE_MAINTENANCE_COSTS = { # units of % / year "light_duty_cars": 11.34, @@ -43,7 +43,7 @@ # maintenance costs in $/mile. This gave very high maintenance costs # when using exogenous lifetime miles assumption. # From https://www.nrel.gov/docs/fy18osti/71500.pdf -# Data in Sheet "A2" at https://data.nrel.gov/submissions/93 +# Data in Sheet "A2" at https://data.nlr.gov/submissions/93 """ ICE_MAINTENANCE_COSTS = { # units of $ / miles "light_duty_cars": 0.033, @@ -96,7 +96,7 @@ class EfsTechnologyData: args: file_path: str - Path to this file https://data.nrel.gov/submissions/78 which is the + Path to this file https://data.nlr.gov/submissions/78 which is the data from this report https://www.nrel.gov/docs/fy18osti/70485.pdf Buildings sector will return both commercial and residential data @@ -254,7 +254,7 @@ def get_capex(self): # noqa: D102 df = df[ (df.Sector == "Transportation") & (df["EFS Case"] == self.efs_case) & (df.Metric == "Capital Cost") ].copy() - source = "NREL EFS at https://data.nrel.gov/submissions/78" + source = "NREL EFS at https://data.nlr.gov/submissions/78" description = "Supplemented with NREL ATB" df["Metric"] = "investment" df = self._format_data_structure(df, source=source, description=description) @@ -278,7 +278,7 @@ def get_efficiency(self): df = df[ (df.Sector == "Transportation") & (df["EFS Case"] == self.efs_case) & (df.Metric == "Main Efficiency") ].copy() - source = "NREL EFS at https://data.nrel.gov/submissions/78" + source = "NREL EFS at https://data.nlr.gov/submissions/78" df["Metric"] = "efficiency" df = self._format_data_structure(df, source=source, description="") df = self._correct_efficiency_units(df) @@ -329,7 +329,7 @@ class EfsIceTransportationData: Only contains ICE vehicles, as this data is manually scraped. - See Table 5 in https://www.nrel.gov/docs/fy18osti/70485.pdf for the sources - - See this file for the raw data https://data.nrel.gov/submissions/93 + - See this file for the raw data https://data.nlr.gov/submissions/93 Args: file_path: str @@ -511,7 +511,7 @@ def get_capex(self): # noqa: D102 & (df.Metric == "Installed Cost") & (df.Units.str.startswith("2016$/kBtu/hr")) ].copy() - source = "NREL EFS at https://data.nrel.gov/submissions/78" + source = "NREL EFS at https://data.nlr.gov/submissions/78" description = "" df["Metric"] = "investment" df = self._format_data_structure(df, source=source, description=description) @@ -529,7 +529,7 @@ def get_lifetime(self): # noqa: D102 def get_efficiency(self): # noqa: D102 df = self.data.copy() df = df[(df.Sector == "Buildings") & (df["EFS Case"] == self.efs_case) & (df.Metric == "Efficiency")].copy() - source = "NREL EFS at https://data.nrel.gov/submissions/78" + source = "NREL EFS at https://data.nlr.gov/submissions/78" df["Metric"] = "efficiency" df["Units"] = "per unit" df = self._format_data_structure(df, source=source, description="") diff --git a/workflow/scripts/build_stock_data.py b/workflow/scripts/build_stock_data.py index c97bb6f07..5a3b966f5 100644 --- a/workflow/scripts/build_stock_data.py +++ b/workflow/scripts/build_stock_data.py @@ -832,7 +832,7 @@ def add_brownfield_lpg( # existing stock efficiencies taken from 2016 EFS Technology data # This is consistent with where future efficiencies are taken from # Historical uses lowest EIA 2017 case where available - # https://data.nrel.gov/submissions/93 + # https://data.nlr.gov/submissions/93 # https://www.nrel.gov/docs/fy18osti/70485.pdf ## Efficiencies of existing stock are quite sensitive! ## diff --git a/workflow/scripts/opts/sector.py b/workflow/scripts/opts/sector.py index 186ff19a7..bb873a3fe 100644 --- a/workflow/scripts/opts/sector.py +++ b/workflow/scripts/opts/sector.py @@ -595,7 +595,7 @@ def add_ev_generation_constraint(n, config, snakemake): Default limits taken from: - (Fig ES2) https://www.nrel.gov/docs/fy18osti/71500.pdf - - (Sheet 6.3 - high case) https://data.nrel.gov/submissions/90 + - (Sheet 6.3 - high case) https://data.nlr.gov/submissions/90 """ mode_mapper = { "light_duty": "lgt", From 344f2bcaf0b8db89a58b2c75acf9c4643556e1a5 Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Fri, 7 Aug 2026 11:49:55 +0200 Subject: [PATCH 05/13] Create parent directories for Zenodo downloads --- workflow/scripts/zenodo_downloader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/workflow/scripts/zenodo_downloader.py b/workflow/scripts/zenodo_downloader.py index 4d7df7d9f..3c9ba43d9 100644 --- a/workflow/scripts/zenodo_downloader.py +++ b/workflow/scripts/zenodo_downloader.py @@ -68,7 +68,10 @@ def download_scenario_file(self, scenario_final, scenario, filename, force_redow force_redownload : bool, optional If True, re-download the file even if it exists locally. Default is False. """ - (self.download_dir / "zenodo" / scenario).mkdir(exist_ok=True) + (self.download_dir / "zenodo" / scenario).mkdir( + parents=True, + exist_ok=True, + ) local_filepath = f"{self.download_dir}/zenodo/{scenario}/{filename}" # Check if file already exists locally and skip Zenodo From 13080b0bd03facd4ff6e864d04d49ff9aa2624f3 Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Fri, 7 Aug 2026 12:01:43 +0200 Subject: [PATCH 06/13] Support archived future renewable profiles --- workflow/scripts/zenodo_downloader.py | 163 +++++++++++++++++--------- 1 file changed, 106 insertions(+), 57 deletions(-) diff --git a/workflow/scripts/zenodo_downloader.py b/workflow/scripts/zenodo_downloader.py index 3c9ba43d9..b7a5a7dee 100644 --- a/workflow/scripts/zenodo_downloader.py +++ b/workflow/scripts/zenodo_downloader.py @@ -1,6 +1,8 @@ """Download scenarios from Zenodo.""" +import shutil from pathlib import Path +from zipfile import BadZipFile, ZipFile import requests @@ -85,12 +87,9 @@ def download_scenario_file(self, scenario_final, scenario, filename, force_redow record_id = self.scenario_records.get(scenario_final) if not record_id: - print(f"No record ID found for scenario: {scenario_final}") - print("Available scenarios with record IDs:") - for scenario, rec_id in self.scenario_records.items(): - if rec_id is not None: - print(f" - {scenario} (ID: {rec_id})") - return None + raise ValueError( + f"No Zenodo record ID configured for scenario: {scenario_final}", + ) return self._download_file(record_id, filename, Path(local_filepath), force_redownload) @@ -147,71 +146,121 @@ def download_by_record_id(self, record_id, filename, force_redownload=False): # Only proceed with download if needed return self._download_file(record_id, filename, Path(local_filepath), force_redownload) - def _download_file(self, record_id, filename, local_filepath, force_redownload=False): - """ - Internal method to download a file from Zenodo. - - This is only called after confirming the file doesn't exist locally. - """ - # Ensure directory exists + def _download_file( + self, + record_id, + filename, + local_filepath, + force_redownload=False, + ): + """Download a file directly or extract it from a Zenodo ZIP archive.""" local_filepath.parent.mkdir(parents=True, exist_ok=True) - # Get record metadata metadata = self.get_record_metadata(record_id) if not metadata: - return None + raise RuntimeError( + f"Could not retrieve metadata for Zenodo record {record_id}", + ) - # Find the specific file - target_file = None - for file_info in metadata.get("files", []): - if file_info["key"] == filename: - target_file = file_info - break - - if not target_file: - print(f"File '{filename}' not found in record {record_id}") - print("Available files:") - for file_info in metadata.get("files", []): - print(f" - {file_info['key']}") - return None + files = metadata.get("files", []) - # Download the file - download_url = target_file["links"]["self"] - file_size_mb = target_file["size"] / (1024 * 1024) + # Some Zenodo records expose the requested NetCDF file directly. + target_file = next( + (file_info for file_info in files if file_info["key"] == filename), + None, + ) - print(f"Downloading {filename} from record {record_id}...") - print(f"Size: {file_size_mb:.1f} MB") - print(f"Saving to: {local_filepath}") + if target_file is not None: + self._download_url( + target_file["links"]["self"], + local_filepath, + target_file["size"], + ) + return str(local_filepath) - try: - response = requests.get(download_url, stream=True) - response.raise_for_status() + # Future renewable datasets are published as ZIP archives. + zip_file = next( + (file_info for file_info in files if file_info["key"].lower().endswith(".zip")), + None, + ) - total_size = int(response.headers.get("content-length", 0)) - downloaded_size = 0 + if zip_file is None: + available_files = ", ".join(file_info["key"] for file_info in files) + raise FileNotFoundError( + f"Neither '{filename}' nor a ZIP archive was found in " + f"Zenodo record {record_id}. Available files: {available_files}", + ) - with open(local_filepath, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - if chunk: - f.write(chunk) - downloaded_size += len(chunk) + archive_path = local_filepath.parent / zip_file["key"] - # Show progress for large files - if total_size > 10 * 1024 * 1024: # Show progress for files > 10MB - progress = (downloaded_size / total_size) * 100 - print(f"\rProgress: {progress:.1f}%", end="", flush=True) + if force_redownload or not archive_path.exists(): + self._download_url( + zip_file["links"]["self"], + archive_path, + zip_file["size"], + ) - if total_size > 10 * 1024 * 1024: - print() # New line after progress + try: + with ZipFile(archive_path) as archive: + matches = [member for member in archive.namelist() if Path(member).name == filename] + + if len(matches) != 1: + raise FileNotFoundError( + f"Expected exactly one '{filename}' in '{archive_path.name}', found {len(matches)}", + ) + + with ( + archive.open(matches[0]) as source, + local_filepath.open("wb") as destination, + ): + shutil.copyfileobj( + source, + destination, + length=1024 * 1024, + ) + + except (BadZipFile, OSError): + local_filepath.unlink(missing_ok=True) + raise + + return str(local_filepath) + + def _download_url(self, download_url, destination, expected_size): + """Download a URL to a local path using a temporary partial file.""" + partial_path = destination.with_suffix( + destination.suffix + ".part", + ) - print(f"Successfully downloaded {filename}") - return str(local_filepath) + print( + f"Downloading {destination.name} ({expected_size / 1024**3:.1f} GiB)...", + ) - except requests.exceptions.RequestException as e: - print(f"Download failed: {e}") - if Path(local_filepath).exists(): - Path(local_filepath).unlink() # Remove partial file - return None + try: + with requests.get( + download_url, + stream=True, + timeout=(30, 300), + ) as response: + response.raise_for_status() + + with partial_path.open("wb") as output: + for chunk in response.iter_content( + chunk_size=1024 * 1024, + ): + if chunk: + output.write(chunk) + + actual_size = partial_path.stat().st_size + if expected_size and actual_size != expected_size: + raise OSError( + f"Incomplete download for {destination.name}: expected {expected_size} bytes, got {actual_size}", + ) + + partial_path.replace(destination) + + except Exception: + partial_path.unlink(missing_ok=True) + raise def list_available_files(self, scenario_name): """List all available files in a scenario dataset.""" From b572a458040631b22ab322fe01858650d66a87ba Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Fri, 7 Aug 2026 14:24:44 +0200 Subject: [PATCH 07/13] Handle renewable profile filenames in Zenodo archives --- workflow/scripts/zenodo_downloader.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/workflow/scripts/zenodo_downloader.py b/workflow/scripts/zenodo_downloader.py index b7a5a7dee..14de9448b 100644 --- a/workflow/scripts/zenodo_downloader.py +++ b/workflow/scripts/zenodo_downloader.py @@ -202,11 +202,15 @@ def _download_file( try: with ZipFile(archive_path) as archive: - matches = [member for member in archive.namelist() if Path(member).name == filename] + archive_filename = filename.replace("_aggregated.nc", ".nc") + + matches = [member for member in archive.namelist() if Path(member).name in {filename, archive_filename}] if len(matches) != 1: raise FileNotFoundError( - f"Expected exactly one '{filename}' in '{archive_path.name}', found {len(matches)}", + f"Expected exactly one of '{filename}' or " + f"'{archive_filename}' in '{archive_path.name}', " + f"found {len(matches)}", ) with ( From bdd52355f09be6381c9e2262f5db2ed7aaba9ed8 Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Fri, 7 Aug 2026 15:34:25 +0200 Subject: [PATCH 08/13] Support archived future renewable profiles --- workflow/scripts/zenodo_downloader.py | 48 +++++++++++++++++++-------- 1 file changed, 34 insertions(+), 14 deletions(-) diff --git a/workflow/scripts/zenodo_downloader.py b/workflow/scripts/zenodo_downloader.py index 14de9448b..1a73d4ad4 100644 --- a/workflow/scripts/zenodo_downloader.py +++ b/workflow/scripts/zenodo_downloader.py @@ -44,7 +44,10 @@ def get_record_metadata(self, record_id): url = f"https://zenodo.org/api/records/{record_id}" try: - response = requests.get(url) + response = requests.get( + url, + timeout=(30, 120), + ) response.raise_for_status() metadata = response.json() self._metadata_cache[record_id] = metadata @@ -202,15 +205,11 @@ def _download_file( try: with ZipFile(archive_path) as archive: - archive_filename = filename.replace("_aggregated.nc", ".nc") - - matches = [member for member in archive.namelist() if Path(member).name in {filename, archive_filename}] + matches = [member for member in archive.namelist() if Path(member).name == filename] if len(matches) != 1: raise FileNotFoundError( - f"Expected exactly one of '{filename}' or " - f"'{archive_filename}' in '{archive_path.name}', " - f"found {len(matches)}", + f"Expected exactly one '{filename}' in '{archive_path.name}', found {len(matches)}", ) with ( @@ -223,7 +222,12 @@ def _download_file( length=1024 * 1024, ) - except (BadZipFile, OSError): + except BadZipFile: + local_filepath.unlink(missing_ok=True) + archive_path.unlink(missing_ok=True) + raise + + except OSError: local_filepath.unlink(missing_ok=True) raise @@ -308,16 +312,32 @@ def get_available_scenarios(self): return available -def download_scenario_file(scenario_final, scenario, filename, download_dir="./data/zenodo"): - """Quick function to download a single file from a scenario.""" +def download_scenario_file( + scenario_final, + scenario, + filename, + download_dir="./data", +): + """Download a single file from a configured scenario.""" downloader = ZenodoScenarioDownloader(download_dir) - return downloader.download_scenario_file(scenario_final, scenario, filename) + return downloader.download_scenario_file( + scenario_final, + scenario, + filename, + ) -def download_by_record_id(record_id, filename, download_dir="./data/zenodo"): - """Quick function to download a file directly by record ID.""" +def download_by_record_id( + record_id, + filename, + download_dir="./data", +): + """Download a file directly using its Zenodo record ID.""" downloader = ZenodoScenarioDownloader(download_dir) - return downloader.download_by_record_id(record_id, filename) + return downloader.download_by_record_id( + record_id, + filename, + ) def list_available_scenarios(): From 5b5fa01043634e1fd43b7755daed0027eb9b6e23 Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Mon, 10 Aug 2026 17:19:46 +0200 Subject: [PATCH 09/13] Manage plotting of capacity for networks with no extendable carriers (historical years) --- workflow/scripts/plot_network_maps.py | 40 +++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/workflow/scripts/plot_network_maps.py b/workflow/scripts/plot_network_maps.py index 3e6c5c9a5..6c89519b3 100644 --- a/workflow/scripts/plot_network_maps.py +++ b/workflow/scripts/plot_network_maps.py @@ -466,9 +466,24 @@ def plot_capacity_map_by_horizon( bus_scale = get_bus_scale(interconnect) if interconnect else 1 line_scale = get_line_scale(interconnect) if interconnect else 1 - if bus_values.empty and kind == "new": + has_new_capacity = ( + bus_values.fillna(0).gt(0).any() + or line_values.fillna(0).gt(0).any() + or link_values.fillna(0).gt(0).any() + ) + + if kind == "new" and not has_new_capacity: + logger.info("No new capacity found; creating an empty new-capacity map.") fig, ax = plt.subplots(figsize=(10, 10)) - ax.text(0.5, 0.5, "No new capacity built", ha="center", va="center", fontsize=14) + ax.text( + 0.5, + 0.5, + "No new capacity built", + ha="center", + va="center", + fontsize=14, + transform=ax.transAxes, + ) ax.set_title(title, fontsize=TITLE_SIZE, pad=20) ax.axis("off") else: @@ -551,6 +566,27 @@ def plot_capacity_map_by_horizon( remove_sector_buses(bus_values).groupby(["bus", "carrier"]).sum() if not bus_values.empty else bus_values ) + has_new_capacity = ( + bus_values.fillna(0).gt(0).any() + or line_values.fillna(0).gt(0).any() + or link_values.fillna(0).gt(0).any() + ) + + if kind == "new" and not has_new_capacity: + logger.info("No new capacity found for horizon %s.", horizon) + ax.text( + 0.5, + 0.5, + "No new capacity built", + ha="center", + va="center", + fontsize=14, + transform=ax.transAxes, + ) + ax.set_title(f"{horizon}", fontsize=TITLE_SIZE) + ax.axis("off") + continue + artifacts = _plot_capacity_on_ax( n, bus_values, From 774aa377785800bc319a4b7a55e305ee379c453d Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Tue, 25 Aug 2026 19:15:35 +0200 Subject: [PATCH 10/13] Improve cutout download reliability --- workflow/scripts/build_cutout.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/workflow/scripts/build_cutout.py b/workflow/scripts/build_cutout.py index 37e3a62f4..085068897 100644 --- a/workflow/scripts/build_cutout.py +++ b/workflow/scripts/build_cutout.py @@ -139,4 +139,8 @@ logging.info(f"Preparing cutout with parameters {cutout_params}.") features = cutout_params.pop("features", None) cutout = atlite.Cutout(snakemake.output[0], **cutout_params) - cutout.prepare(features=features) + cutout.prepare( + features=features, + monthly_requests=True, + concurrent_requests=False, + ) From 6d05f68055e5b5ee95778675e06f1c1a1ca10084 Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Tue, 25 Aug 2026 19:15:40 +0200 Subject: [PATCH 11/13] Fix natural gas data year selection --- workflow/scripts/build_natural_gas.py | 2 +- workflow/scripts/eia.py | 2 +- workflow/scripts/opts/sector.py | 6 ++++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/workflow/scripts/build_natural_gas.py b/workflow/scripts/build_natural_gas.py index 10cde0b04..f2669c1e4 100644 --- a/workflow/scripts/build_natural_gas.py +++ b/workflow/scripts/build_natural_gas.py @@ -853,7 +853,7 @@ def _expand_costs(self, n: pypsa.Network, costs: pd.DataFrame) -> pd.DataFrame: expanded_costs = [] for invesetment_period in n.investment_periods: # reindex to match any tsa - cost = costs.copy() + cost = costs.loc[costs.index.year == self.year].copy() cost.index = cost.index.map(lambda x: x.replace(year=invesetment_period)) cost = cost.reindex(n.snapshots.get_level_values(1), method="nearest") # set investment periods diff --git a/workflow/scripts/eia.py b/workflow/scripts/eia.py index 5d27c9145..3a264e77b 100644 --- a/workflow/scripts/eia.py +++ b/workflow/scripts/eia.py @@ -1607,7 +1607,7 @@ def extract_state(description: str) -> str: @staticmethod def map_state_names(state: str) -> str: """Maps state name to code.""" - return "U.S." if state == "U.S. Total" else STATE_CODES[state] + return "U.S." if state in {"U.S.", "U.S. Total"} else STATE_CODES[state] class _GasProduction(DataExtractor): diff --git a/workflow/scripts/opts/sector.py b/workflow/scripts/opts/sector.py index bb873a3fe..1a1f05e9f 100644 --- a/workflow/scripts/opts/sector.py +++ b/workflow/scripts/opts/sector.py @@ -420,7 +420,8 @@ def add_export_limits(n, data, constraint, multiplier=None): # add domestic limits trade = Trade("gas", False, "exports", year, api).get_data() - trade = _format_data(trade, " trade") + trade = trade.loc[trade.index == year] + trade = _format_data(trade, " trade").groupby(level=0).sum() add_import_limits(n, trade, "min", import_min) add_export_limits(n, trade, "min", export_min) @@ -433,7 +434,8 @@ def add_export_limits(n, data, constraint, multiplier=None): # add international limits trade = Trade("gas", True, "exports", year, api).get_data() - trade = _format_data(trade, " trade") + trade = trade.loc[trade.index == year] + trade = _format_data(trade, " trade").groupby(level=0).sum() add_import_limits(n, trade, "min", import_min) add_export_limits(n, trade, "min", export_min) From 7a8aab99b610d822a177e21fe641b2e47fb2acb6 Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Tue, 25 Aug 2026 20:39:37 +0200 Subject: [PATCH 12/13] Pre-commit checks --- workflow/rules/build_electricity.smk | 1 + workflow/scripts/plot_network_maps.py | 8 ++------ 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/workflow/rules/build_electricity.smk b/workflow/rules/build_electricity.smk index e20fa930d..2c1c68d41 100644 --- a/workflow/rules/build_electricity.smk +++ b/workflow/rules/build_electricity.smk @@ -222,6 +222,7 @@ rule build_cost_data: script: "../scripts/build_cost_data.py" + ATLITE_NPROCESSES = config["atlite"].get("nprocesses", 4) if config["enable"].get("build_cutout", False): diff --git a/workflow/scripts/plot_network_maps.py b/workflow/scripts/plot_network_maps.py index 6c89519b3..55b54bab9 100644 --- a/workflow/scripts/plot_network_maps.py +++ b/workflow/scripts/plot_network_maps.py @@ -467,9 +467,7 @@ def plot_capacity_map_by_horizon( line_scale = get_line_scale(interconnect) if interconnect else 1 has_new_capacity = ( - bus_values.fillna(0).gt(0).any() - or line_values.fillna(0).gt(0).any() - or link_values.fillna(0).gt(0).any() + bus_values.fillna(0).gt(0).any() or line_values.fillna(0).gt(0).any() or link_values.fillna(0).gt(0).any() ) if kind == "new" and not has_new_capacity: @@ -567,9 +565,7 @@ def plot_capacity_map_by_horizon( ) has_new_capacity = ( - bus_values.fillna(0).gt(0).any() - or line_values.fillna(0).gt(0).any() - or link_values.fillna(0).gt(0).any() + bus_values.fillna(0).gt(0).any() or line_values.fillna(0).gt(0).any() or link_values.fillna(0).gt(0).any() ) if kind == "new" and not has_new_capacity: From 99a189bb6c4bba56f5e7bba4570bdffc2b64feaf Mon Sep 17 00:00:00 2001 From: Daniele Lerede Date: Tue, 25 Aug 2026 20:52:37 +0200 Subject: [PATCH 13/13] Fix PUDL configuration and local imports --- workflow/config/config.common.yaml | 4 ++++ workflow/rules/common.smk | 7 +++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/workflow/config/config.common.yaml b/workflow/config/config.common.yaml index 8fafacbb8..1d8d0de58 100644 --- a/workflow/config/config.common.yaml +++ b/workflow/config/config.common.yaml @@ -1,3 +1,7 @@ +pudl_cache: + version: v2025.5.0 + base_url: https://s3.us-west-2.amazonaws.com/pudl.catalyst.coop + directory: repo_data/pudl pudl_path: s3://pudl.catalyst.coop/v2025.2.0 foresight: 'perfect' diff --git a/workflow/rules/common.smk b/workflow/rules/common.smk index 3ddae2eb6..07295d0eb 100644 --- a/workflow/rules/common.smk +++ b/workflow/rules/common.smk @@ -7,8 +7,11 @@ from functools import partial, lru_cache import os, sys, glob -path = workflow.source_path("../scripts/_helpers.py") -sys.path.insert(0, os.path.dirname(path)) +# Cache local modules before importing them from Snakemake's source cache. +for source in ("../scripts/_helpers.py", "../scripts/constants.py"): + source_dir = os.path.dirname(workflow.source_path(source)) + if source_dir not in sys.path: + sys.path.insert(0, source_dir) from _helpers import validate_checksum, update_config_from_wildcards from constants import HOURS_PER_YEAR