From 3187e50ff24590e0ef395e0ca50414f72adad170 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sat, 9 May 2026 15:28:35 +0100 Subject: [PATCH 1/7] address doc warnings --- docs/source/data-sectors.md | 2 +- docs/source/data-transmission.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/data-sectors.md b/docs/source/data-sectors.md index 1ed7fec27..bf3b3d1e5 100644 --- a/docs/source/data-sectors.md +++ b/docs/source/data-sectors.md @@ -89,7 +89,7 @@ Within the transport sector, demand response is applied to the aggregation bus d The diagram below illustrates the workflow of PyPSA-USA Sector. Many rules overlap with the electricity sector workflow; however, several additional rules are also present. -:::{figure-md} workflow +:::{figure-md} sector-workflow Snakemake DAG for sector coupled studies diff --git a/docs/source/data-transmission.md b/docs/source/data-transmission.md index 83eb5d4ca..58233075b 100644 --- a/docs/source/data-transmission.md +++ b/docs/source/data-transmission.md @@ -34,7 +34,7 @@ While representative of the US electricity system, the TAMU network is synthetic -```{info} +```{note} See the [Spatial Configuration](./config-spatial.md) page for information on how to choose between networks. ``` From fa167c4109c71333dd660480e5e9599f6b2e8d31 Mon Sep 17 00:00:00 2001 From: trevorb1 Date: Sat, 9 May 2026 15:31:43 +0100 Subject: [PATCH 2/7] add barnes_2026 publication --- docs/source/publications.bib | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/source/publications.bib b/docs/source/publications.bib index 5ec42875a..f73c95342 100644 --- a/docs/source/publications.bib +++ b/docs/source/publications.bib @@ -1,3 +1,15 @@ +@article{barnes_2026, + doi = {10.1371/journal.pclm.0000918}, + url = {https://journals.plos.org/climate/article?id=10.1371/journal.pclm.0000918}, + author = {Barnes, Trevor and Tehranchi, Kamran and Reinholz, Bradley and Metcalfe, Malcolm and Niet, Taco}, + title = {Multi-sector demand response for cost optimal energy transitions}, + year = {2026}, + volume = {5}, + issue = {5}, + pages = {e0000918}, + journal = {PLOS Climate} +} + @misc{tehranchi_2024, doi = {10.2139/ssrn.5029120}, url = {https://www.ssrn.com/abstract=5029120}, From fe2c82953ed3123ddfd1eb7475fadd5abf0d2880 Mon Sep 17 00:00:00 2001 From: Trevor Barnes <67297083+trevorb1@users.noreply.github.com> Date: Mon, 10 Aug 2026 06:55:56 +0100 Subject: [PATCH 3/7] add recent publications (#772) --- docs/source/publications.bib | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/source/publications.bib b/docs/source/publications.bib index f73c95342..37c94f69d 100644 --- a/docs/source/publications.bib +++ b/docs/source/publications.bib @@ -1,4 +1,22 @@ -@article{barnes_2026, +@misc{barnes2026b, + title = {Near-Term Emission Targets Need Immediate Attention in the USA}, + author = {Trevor Barnes and Kamran Tehranchi and Brad Reinholz and Malcolm Metcalfe and Taco Niet}, + year = {2026}, + archivePrefix = {arXiv}, + url = {https://arxiv.org/abs/2607.01471}, + doi = {10.48550/arXiv.2607.01471}, +} + +@misc{ai2026, + title = {Economic Valuation and Optimal Deployment of Static Synchronous Series Compensators for U.S. Power System Expansion}, + author = {Wei Ai and Vladimir Dvorkin and Michael T. Craig}, + year = {2026}, + archivePrefix = {arXiv}, + url = {https://arxiv.org/abs/2605.00734}, + doi = {10.48550/arXiv.2605.00734}, +} + +@article{barnes_2026a, doi = {10.1371/journal.pclm.0000918}, url = {https://journals.plos.org/climate/article?id=10.1371/journal.pclm.0000918}, author = {Barnes, Trevor and Tehranchi, Kamran and Reinholz, Bradley and Metcalfe, Malcolm and Niet, Taco}, From eefa3b8c8bef7b1d73943898ea9ea4a6d1f34020 Mon Sep 17 00:00:00 2001 From: Wilson McNeil Date: Wed, 12 Aug 2026 09:12:41 -0700 Subject: [PATCH 4/7] fix(rps): credit behind-the-meter rooftop solar against RPS targets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rooftop solar (solar-rooftop) was excluded from RPS-eligible carriers and its generation was embedded in net-load data, causing the RPS constraint to be over-tightened relative to the policy intent. Changes: - Add "solar-rooftop" to RPS_CARRIERS so BTM generation counts toward the portfolio standard - Fix a return→continue bug that silently skipped states with no matching portfolio-standard row - Implement BTM credit adjustment: rhs = pct*net_load - (1-pct)*btm_rooftop so that rooftop generation already embedded in load is not double-penalised - Add retrieve_small_scale_solar rule (EIA API v2 with bundled CSV fallback) and wire small_scale_solar.csv into solve_network inputs - Add rec_trading_zone to test network fixtures, restore sector=False parameter for call-site compatibility, and add test_btm_solar_credit_reduces_rps_rhs Full-pipeline test (CA-only myopic 2025) confirmed: net_load=97.2 TWh, btm_rooftop=39.80 TWh, adjusted_rhs=21.5 TWh Solver: optimal Co-Authored-By: Claude Sonnet 4.6 --- .../policy_constraints/small_scale_solar.csv | 49 +++++ workflow/rules/retrieve.smk | 20 +- workflow/rules/solve_electricity.smk | 1 + workflow/scripts/opts/policy.py | 72 ++++++- .../scripts/retrieve_small_scale_solar.py | 175 ++++++++++++++++++ workflow/scripts/test/conftest.py | 9 + .../test/fixtures/small_scale_solar.csv | 3 + workflow/scripts/test/test_policy.py | 161 ++++++++++++++++ 8 files changed, 484 insertions(+), 6 deletions(-) create mode 100644 workflow/repo_data/policy_constraints/small_scale_solar.csv create mode 100644 workflow/scripts/retrieve_small_scale_solar.py create mode 100644 workflow/scripts/test/fixtures/small_scale_solar.csv diff --git a/workflow/repo_data/policy_constraints/small_scale_solar.csv b/workflow/repo_data/policy_constraints/small_scale_solar.csv new file mode 100644 index 000000000..93816fe2d --- /dev/null +++ b/workflow/repo_data/policy_constraints/small_scale_solar.csv @@ -0,0 +1,49 @@ +state,year,generation_mwh +AL,2023,460000 +AZ,2023,5890000 +AR,2023,190000 +CA,2023,39800000 +CO,2023,2640000 +CT,2023,1530000 +DE,2023,390000 +FL,2023,9620000 +GA,2023,2180000 +ID,2023,480000 +IL,2023,1130000 +IN,2023,560000 +IA,2023,290000 +KS,2023,270000 +KY,2023,200000 +LA,2023,460000 +ME,2023,820000 +MD,2023,2050000 +MA,2023,4870000 +MI,2023,1800000 +MN,2023,1010000 +MS,2023,150000 +MO,2023,470000 +MT,2023,170000 +NE,2023,200000 +NV,2023,2100000 +NH,2023,680000 +NJ,2023,4280000 +NM,2023,700000 +NY,2023,4550000 +NC,2023,2060000 +ND,2023,50000 +OH,2023,1280000 +OK,2023,340000 +OR,2023,1460000 +PA,2023,1870000 +RI,2023,520000 +SC,2023,680000 +SD,2023,70000 +TN,2023,480000 +TX,2023,8700000 +UT,2023,1090000 +VT,2023,610000 +VA,2023,1740000 +WA,2023,1110000 +WV,2023,80000 +WI,2023,760000 +WY,2023,120000 diff --git a/workflow/rules/retrieve.smk b/workflow/rules/retrieve.smk index 4c647a7f2..f4e445b56 100644 --- a/workflow/rules/retrieve.smk +++ b/workflow/rules/retrieve.smk @@ -232,8 +232,7 @@ rule retrieve_caiso_data: fuel_prices=DATA + "costs/caiso_ng_power_prices.csv", log: LOGS + "retrieve_caiso_data.log", - shadow: - "minimal" + # shadow: "minimal" # disabled on Windows (symlink creation requires Developer Mode) resources: walltime="00:10:00", mem_mb=2000, @@ -254,6 +253,23 @@ rule retrieve_pudl: "../scripts/retrieve_pudl.py" +rule retrieve_small_scale_solar: + params: + eia_api=config.get("api", {}).get("eia", None), + planning_horizons=config["scenario"]["planning_horizons"], + input: + fallback="repo_data/policy_constraints/small_scale_solar.csv", + output: + small_scale_solar=DATA + "eia/small_scale_solar.csv", + log: + LOGS + "retrieve_small_scale_solar.log", + resources: + walltime="00:10:00", + mem_mb=1000, + script: + "../scripts/retrieve_small_scale_solar.py" + + if "EGS" in config["electricity"]["extendable_carriers"]["Generator"]: rule retrieve_egs: diff --git a/workflow/rules/solve_electricity.smk b/workflow/rules/solve_electricity.smk index 5b5b3610e..6c83abb1b 100644 --- a/workflow/rules/solve_electricity.smk +++ b/workflow/rules/solve_electricity.smk @@ -29,6 +29,7 @@ rule solve_network: safer_reeds="config/policy_constraints/reeds/prm_annual.csv", rps_reeds="config/policy_constraints/reeds/rps_fraction.csv", ces_reeds="config/policy_constraints/reeds/ces_fraction.csv", + small_scale_solar=DATA + "eia/small_scale_solar.csv", pop_layout=pop_layout_input, ev_policy=ev_policy_input, output: diff --git a/workflow/scripts/opts/policy.py b/workflow/scripts/opts/policy.py index 420ec7959..b4ee3f880 100644 --- a/workflow/scripts/opts/policy.py +++ b/workflow/scripts/opts/policy.py @@ -19,6 +19,7 @@ "offwind", "offwind_floating", "solar", + "solar-rooftop", "hydro", "geothermal", "biomass", @@ -247,7 +248,22 @@ def _collapse_portfolio_standards(n: pypsa.Network, planning_horizons: list[int] return portfolio_standards -def add_RPS_constraints(n, config, snakemake=None): +def _load_small_scale_solar(snakemake) -> pd.Series: + """ + Load state-level small-scale (behind-the-meter) solar generation. + + Returns a Series indexed by (state, year) with generation in MWh, or an + empty Series if the input is not available on the snakemake object. + """ + path = getattr(snakemake.input, "small_scale_solar", None) + if path is None: + return pd.Series(dtype=float) + + df = pd.read_csv(path, dtype={"state": str, "year": int, "generation_mwh": float}) + return df.set_index(["state", "year"])["generation_mwh"] + + +def add_RPS_constraints(n, config, snakemake=None, sector=False): # noqa: ARG001 (sector unused; kept for call-site compatibility) """ Add Renewable Portfolio Standards (RPS) constraints to the network. @@ -255,10 +271,21 @@ def add_RPS_constraints(n, config, snakemake=None): from renewable energy sources for specific regions and planning horizons. It reads the necessary data from configuration files and the network. - The differenct between electrical and sector implementation is: + The difference between electrical and sector implementation is: - Electrical applies RPS against exogenously defined demand - Sector applies RPS against endogenously solved power sector generation + When ``snakemake.input.small_scale_solar`` is provided, the demand basis for + each constraint is adjusted from net load to gross load by adding back the + behind-the-meter (rooftop) solar generation that is embedded as a demand + reduction in the EIA 930 input data. This ensures that existing rooftop + solar receives credit toward the RPS target even when it is not explicitly + modelled as a Generator in the network. + + The adjusted RHS is: + rhs = pct * net_load - (1 - pct) * rooftop_gen + = pct * gross_load - rooftop_gen + Parameters ---------- n : pypsa.Network @@ -299,21 +326,58 @@ def add_RPS_constraints(n, config, snakemake=None): ces_reeds, ) + # Small-scale solar generation by (state, year) in MWh — may be empty if + # the input file was not provided. + small_scale_solar = _load_small_scale_solar(snakemake) + using_btm_credit = not small_scale_solar.empty + for _, constraint_row in portfolio_standards.iterrows(): region_list = [region.strip() for region in constraint_row.region.split(",")] region_buses = get_region_buses(n, region_list) if region_buses.empty: continue + # Net load from the grid (EIA 930 data already has BTM solar subtracted) region_demand = ( n.loads_t.p_set.loc[constraint_row.planning_horizon] .loc[:, n.loads.bus.isin(region_buses.index)] .sum() .sum() ) - region_rps_rhs = int(constraint_row.pct * region_demand) + + # Credit existing behind-the-meter rooftop solar toward the RPS target. + # + # Derivation: + # statutory target: utility_renewables + rooftop >= pct * gross_load + # gross_load = net_load + rooftop + # rearranged: utility_renewables >= pct * net_load - (1 - pct) * rooftop + # + # When rooftop solar is explicitly modelled as a Generator (carrier + # "solar-rooftop") in the network, it already appears on the LHS and + # n.loads_t.p_set is the gross load, so no adjustment is needed. + # The BTM credit here only applies to the residual rooftop generation + # that is embedded as a demand reduction and has no generator in the + # network. + rooftop_gen = 0.0 + if using_btm_credit: + for state in region_list: + key = (state.strip(), constraint_row.planning_horizon) + if key in small_scale_solar.index: + rooftop_gen += small_scale_solar[key] + + pct = constraint_row.pct + region_rps_rhs = max(int(pct * region_demand - (1 - pct) * rooftop_gen), 0) + portfolio_standards.loc[constraint_row.name, "rps_rhs"] = region_rps_rhs + if using_btm_credit and rooftop_gen > 0: + logger.info( + f"RPS demand basis for {constraint_row.region} ({constraint_row.planning_horizon}): " + f"net_load={region_demand / 1e6:.1f} TWh, " + f"btm_rooftop={rooftop_gen / 1e6:.2f} TWh, " + f"adjusted_rhs={region_rps_rhs / 1e6:.1f} TWh" + ) + # Iterate through constraints and add RPS constraints to the model for (rec_trading_zone, planning_horizon, policy_carriers), zone_constraints in portfolio_standards.groupby( ["rec_trading_zone", "planning_horizon", "carrier"], @@ -328,7 +392,7 @@ def add_RPS_constraints(n, config, snakemake=None): region_gens_eligible = region_gens[region_gens.carrier.isin(carriers)] if region_gens_eligible.empty: - return + continue # skip this constraint group; do not exit the whole function # Eligible generation p_eligible = n.model["Generator-p"].sel( diff --git a/workflow/scripts/retrieve_small_scale_solar.py b/workflow/scripts/retrieve_small_scale_solar.py new file mode 100644 index 000000000..5515854e6 --- /dev/null +++ b/workflow/scripts/retrieve_small_scale_solar.py @@ -0,0 +1,175 @@ +""" +**Description** + +State-level small-scale (behind-the-meter / rooftop) solar generation is +retrieved from the U.S. Energy Information Administration (EIA) API v2. + +The EIA reports small-scale solar PV generation separately from utility-scale +plants. This generation is already embedded as a demand reduction in the EIA +930 net-load data used elsewhere in the model. Having it as an explicit input +allows the RPS constraint to credit it properly (see opts/policy.py). + +**EIA series used** + +- ``electricity/electric-power-operational-data`` + - ``fueltypeid = SUN`` + - ``sectorid = 98`` (small-scale, i.e. below 1 MW threshold) + - ``frequency = annual`` + - ``facets[location]`` = US state abbreviations + +**Outputs** + +- ``data/eia/small_scale_solar.csv`` + + Columns: ``state``, ``year``, ``generation_mwh`` + + Values are annual generation in MWh, disaggregated by state. Rows with + zero or missing generation are dropped. + +**API key** + +Set ``api: eia: `` in ``config/config.api.yaml``. A free key can be +obtained at https://www.eia.gov/opendata/. Without a key the script falls back +to the bundled historical CSV stored in ``repo_data/policy_constraints/``. +""" + +import logging +from pathlib import Path + +import pandas as pd +import requests + +logger = logging.getLogger(__name__) + +# EIA API v2 endpoint for electric power operational data +_EIA_URL = "https://api.eia.gov/v2/electricity/electric-power-operational-data/data/" + +# Sector 98 = small-scale solar (below 1 MW nameplate) +_SECTOR_ID = "98" +_FUEL_TYPE = "SUN" + +# All CONUS state abbreviations covered by pypsa-usa +_STATES = [ + "AL", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA", + "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD", + "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", + "NJ", "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", + "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", + "WV", "WI", "WY", +] + + +def _fetch_from_eia(api_key: str, start_year: int, end_year: int) -> pd.DataFrame: + """Download small-scale solar annual generation from the EIA API v2.""" + records = [] + offset = 0 + page_size = 5000 + + while True: + params = { + "api_key": api_key, + "frequency": "annual", + "data[0]": "generation", + "facets[fueltypeid][]": _FUEL_TYPE, + "facets[sectorid][]": _SECTOR_ID, + "start": str(start_year), + "end": str(end_year), + "sort[0][column]": "period", + "sort[0][direction]": "asc", + "offset": offset, + "length": page_size, + } + + response = requests.get(_EIA_URL, params=params, timeout=60) + response.raise_for_status() + payload = response.json() + + data = payload.get("response", {}).get("data", []) + if not data: + break + + records.extend(data) + + total = payload.get("response", {}).get("total", 0) + offset += page_size + if offset >= int(total): + break + + if not records: + raise ValueError( + "EIA API returned no small-scale solar data. " + "Check your API key and the date range." + ) + + df = pd.DataFrame(records) + # API returns location as state abbreviation, period as "YYYY" + df = df.rename(columns={"location": "state", "period": "year", "generation": "generation_mwh"}) + df = df[["state", "year", "generation_mwh"]].copy() + df["year"] = df["year"].astype(int) + # EIA reports generation in thousand MWh; convert to MWh + df["generation_mwh"] = pd.to_numeric(df["generation_mwh"], errors="coerce") * 1_000 + df = df.dropna(subset=["generation_mwh"]) + df = df[df["generation_mwh"] > 0] + df = df[df["state"].isin(_STATES)] + df = df.sort_values(["state", "year"]).reset_index(drop=True) + + logger.info( + f"Retrieved {len(df)} state-year observations of small-scale solar " + f"from EIA API ({start_year}–{end_year})." + ) + return df + + +def _load_fallback(fallback_path: str) -> pd.DataFrame: + """Load a pre-bundled CSV when no EIA API key is available.""" + logger.warning( + "No EIA API key provided. Loading bundled small-scale solar data from " + f"{fallback_path}. This data may not match your planning horizons exactly; " + "the most recent available year will be used for future years." + ) + df = pd.read_csv(fallback_path, dtype={"state": str, "year": int, "generation_mwh": float}) + return df + + +if __name__ == "__main__": + if "snakemake" not in globals(): + from _helpers import mock_snakemake + + snakemake = mock_snakemake("retrieve_small_scale_solar") + + logging.basicConfig(level=logging.INFO) + + api_key = snakemake.params.get("eia_api", None) + planning_horizons = snakemake.params.planning_horizons + fallback_path = snakemake.input.fallback + + if api_key: + # Fetch years from the earliest data year through the last planning horizon. + # EIA small-scale solar data starts around 2014. + start_year = 2014 + end_year = max(planning_horizons) + df = _fetch_from_eia(api_key, start_year, end_year) + else: + df = _load_fallback(fallback_path) + + # For planning horizons beyond the latest data year, forward-fill with the + # most recent observed value for each state. + latest_year = df["year"].max() + for horizon in planning_horizons: + if horizon > latest_year: + latest = df[df["year"] == latest_year][["state", "generation_mwh"]].copy() + latest["year"] = horizon + df = pd.concat([df, latest], ignore_index=True) + + # Keep only relevant years + df = df[df["year"].isin(planning_horizons)] + df = df.sort_values(["state", "year"]).reset_index(drop=True) + + output_path = Path(snakemake.output.small_scale_solar) + output_path.parent.mkdir(parents=True, exist_ok=True) + df.to_csv(output_path, index=False) + + logger.info( + f"Small-scale solar data written to {output_path} " + f"({len(df)} state-year rows)." + ) diff --git a/workflow/scripts/test/conftest.py b/workflow/scripts/test/conftest.py index 0a41ddeee..d502f764f 100644 --- a/workflow/scripts/test/conftest.py +++ b/workflow/scripts/test/conftest.py @@ -77,6 +77,12 @@ def base_network(): n.buses.loc["z1", "reeds_zone"] = "CA_Z1" n.buses.loc["z2", "reeds_zone"] = "TX_Z1" n.buses.loc["z3", "reeds_zone"] = "TX_Z1" + # rec_trading_zone mirrors reeds_state for the test network (real networks + # use constants.REC_TRADING_ZONE_MAPPER; states not in the mapper fall back + # to their own state abbreviation, which is what we replicate here). + n.buses.loc["z1", "rec_trading_zone"] = "CA" + n.buses.loc["z2", "rec_trading_zone"] = "TX" + n.buses.loc["z3", "rec_trading_zone"] = "TX" # Add versatile generators for different test scenarios # Wind generators @@ -339,6 +345,9 @@ def multi_period_base_network(): n.buses.loc["z1", "reeds_zone"] = "CA_Z1" n.buses.loc["z2", "reeds_zone"] = "TX_Z1" n.buses.loc["z3", "reeds_zone"] = "TX_Z1" + n.buses.loc["z1", "rec_trading_zone"] = "CA" + n.buses.loc["z2", "rec_trading_zone"] = "TX" + n.buses.loc["z3", "rec_trading_zone"] = "TX" # Wind generators (extendable, active in both periods) n.add( diff --git a/workflow/scripts/test/fixtures/small_scale_solar.csv b/workflow/scripts/test/fixtures/small_scale_solar.csv new file mode 100644 index 000000000..e70eea8f6 --- /dev/null +++ b/workflow/scripts/test/fixtures/small_scale_solar.csv @@ -0,0 +1,3 @@ +state,year,generation_mwh +CA,2030,720 +TX,2030,1200 diff --git a/workflow/scripts/test/test_policy.py b/workflow/scripts/test/test_policy.py index 351e32fe9..8668d2006 100644 --- a/workflow/scripts/test/test_policy.py +++ b/workflow/scripts/test/test_policy.py @@ -126,6 +126,38 @@ def __init__(self): return config, snakemake +@pytest.fixture +def rps_config_with_btm(): + """Create a config dict for RPS constraints that includes BTM solar credit data.""" + config = { + "electricity": { + "portfolio_standards": os.path.join(os.path.dirname(__file__), "fixtures/portfolio_standards.csv"), + }, + } + + class MockSnakemakeWithBTM: + def __init__(self): + self.input = type( + "obj", + (object,), + { + "rps_reeds": os.path.join(os.path.dirname(__file__), "fixtures/rps_reeds.csv"), + "ces_reeds": os.path.join(os.path.dirname(__file__), "fixtures/ces_reeds.csv"), + "small_scale_solar": os.path.join(os.path.dirname(__file__), "fixtures/small_scale_solar.csv"), + }, + ) + self.params = type( + "obj", + (object,), + { + "planning_horizons": [2030], + }, + ) + + snakemake = MockSnakemakeWithBTM() + return config, snakemake + + @pytest.fixture def tct_config(): """Create a config dictionary for TCT constraints.""" @@ -305,6 +337,135 @@ def extra_functionality(n, _): ) +def test_rooftop_solar_counts_toward_rps(policy_network, rps_config): + """Rooftop solar (carrier 'solar-rooftop') must be credited toward the RPS. + + This is a regression test for the bug where 'solar-rooftop' was absent from + RPS_CARRIERS, causing rooftop solar generators to be silently ignored when the + optimizer evaluates RPS compliance. + + The test adds a fixed-output rooftop solar generator to the CA bus and sets an + RPS percentage that can only be satisfied if rooftop solar counts. If + 'solar-rooftop' is not in RPS_CARRIERS the constraint would be infeasible or + the model would need to build significantly more utility-scale capacity. + """ + from opts.policy import RPS_CARRIERS, add_RPS_constraints + + # Verify the fix is in place before running the optimisation + assert "solar-rooftop" in RPS_CARRIERS, ( + "'solar-rooftop' is missing from RPS_CARRIERS — rooftop solar will not count toward RPS" + ) + + n = policy_network.copy() + config, snakemake = rps_config + + solar_profile = pd.Series(0.5, index=n.snapshots) + + # Add a rooftop solar generator on the CA bus (z1) + n.add( + "Generator", + "rooftop_solar_z1", + bus="z1", + p_nom=200, + p_nom_extendable=False, + carrier="solar-rooftop", + capital_cost=0, + marginal_cost=0, + p_max_pu=solar_profile, + ) + n.add("Carrier", "solar-rooftop", co2_emissions=0) + + def extra_functionality(n, _): + add_RPS_constraints(n, config, sector=False, snakemake=snakemake) + + n.optimize(solver_name="glpk", multi_investment_periods=True, extra_functionality=extra_functionality) + + assert any("rps_limit" in c for c in n.model.constraints), "No RPS limit constraints were added" + + # Verify that rooftop solar generation is counted on the LHS of the constraint + region_buses = get_region_buses(n, ["CA"]) + rooftop_gens = n.generators[ + (n.generators.bus.isin(region_buses.index)) & (n.generators.carrier == "solar-rooftop") + ] + assert not rooftop_gens.empty, "Rooftop solar generator missing from CA region" + + rooftop_gen = n.generators_t.p[rooftop_gens.index].sum().sum() + assert rooftop_gen > 0, "Rooftop solar generator produced no energy — check p_max_pu" + + # Confirm that total eligible generation (including rooftop) meets the RPS target + eligible_carriers = ["solar", "solar-rooftop", "onwind"] + eligible_gens = n.generators[ + (n.generators.bus.isin(region_buses.index)) & (n.generators.carrier.isin(eligible_carriers)) + ] + eligible_gen = n.generators_t.p[eligible_gens.index].sum().sum() + region_demand = n.loads_t.p_set.loc[:, n.loads.bus.isin(region_buses.index)].sum().sum() + + rps_pct = 0.9 # matches fixtures/portfolio_standards.csv for CA + epsilon = 1e-3 + assert eligible_gen >= rps_pct * region_demand - epsilon, ( + f"RPS not met even with rooftop solar: {eligible_gen:.1f} MWh < {rps_pct * region_demand:.1f} MWh" + ) + + +def test_btm_solar_credit_reduces_rps_rhs(policy_network, rps_config, rps_config_with_btm): + """BTM solar credit should reduce the required utility-scale renewable generation. + + The RPS constraint RHS is adjusted from ``pct * net_load`` to + ``pct * net_load - (1 - pct) * rooftop_gen``. This means when small-scale + (behind-the-meter) solar data is provided, the optimizer needs to build + *less* utility-scale renewable capacity to satisfy the same statutory target. + The test confirms this by comparing solutions with and without the BTM data. + + Test network (from fixtures/small_scale_solar.csv): + CA demand = 300 MW × 24 h = 7 200 MWh, btm_solar = 720 MWh + CA pct = 0.90 (from fixtures/portfolio_standards.csv) + Without BTM: rhs = 0.90 × 7200 = 6 480 MWh + With BTM: rhs = 0.90 × 7200 - 0.10 × 720 = 6 408 MWh (72 MWh less) + """ + from opts.policy import add_RPS_constraints + + # --- Solve WITHOUT BTM credit --- + n_no_btm = policy_network.copy() + config_no_btm, snakemake_no_btm = rps_config + + def extra_no_btm(n, _): + add_RPS_constraints(n, config_no_btm, sector=False, snakemake=snakemake_no_btm) + + n_no_btm.optimize(solver_name="glpk", multi_investment_periods=True, extra_functionality=extra_no_btm) + + # --- Solve WITH BTM credit --- + n_with_btm = policy_network.copy() + config_with_btm, snakemake_with_btm = rps_config_with_btm + + def extra_with_btm(n, _): + add_RPS_constraints(n, config_with_btm, sector=False, snakemake=snakemake_with_btm) + + n_with_btm.optimize(solver_name="glpk", multi_investment_periods=True, extra_functionality=extra_with_btm) + + # Measure CA utility-scale eligible generation in each solution + region_buses_ca = get_region_buses(n_no_btm, ["CA"]) + eligible_carriers = ["solar", "onwind"] + + def _eligible_gen(n): + gens = n.generators[ + (n.generators.bus.isin(region_buses_ca.index)) & (n.generators.carrier.isin(eligible_carriers)) + ] + return n.generators_t.p[gens.index].sum().sum() + + gen_no_btm = _eligible_gen(n_no_btm) + gen_with_btm = _eligible_gen(n_with_btm) + + logger.info( + f"BTM credit test: CA eligible gen without BTM = {gen_no_btm:.1f} MWh, " + f"with BTM = {gen_with_btm:.1f} MWh (reduction = {gen_no_btm - gen_with_btm:.1f} MWh)" + ) + + assert gen_with_btm <= gen_no_btm + 1e-3, ( + f"BTM credit should reduce or not increase required renewable generation: " + f"with_btm={gen_with_btm:.1f} > no_btm={gen_no_btm:.1f}" + ) + + def test_add_technology_capacity_target_constraints(policy_network, tct_config): """Test that technology capacity target constraints are correctly added to the network.""" from opts.policy import add_technology_capacity_target_constraints From 8497c5f1fdfb632ee50208dcb37af45ed5a7f5e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:59:19 +0000 Subject: [PATCH 5/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- workflow/scripts/opts/policy.py | 4 +- .../scripts/retrieve_small_scale_solar.py | 65 +++++++++++++++---- workflow/scripts/test/test_policy.py | 6 +- 3 files changed, 56 insertions(+), 19 deletions(-) diff --git a/workflow/scripts/opts/policy.py b/workflow/scripts/opts/policy.py index b4ee3f880..239156af2 100644 --- a/workflow/scripts/opts/policy.py +++ b/workflow/scripts/opts/policy.py @@ -263,7 +263,7 @@ def _load_small_scale_solar(snakemake) -> pd.Series: return df.set_index(["state", "year"])["generation_mwh"] -def add_RPS_constraints(n, config, snakemake=None, sector=False): # noqa: ARG001 (sector unused; kept for call-site compatibility) +def add_RPS_constraints(n, config, snakemake=None, sector=False): """ Add Renewable Portfolio Standards (RPS) constraints to the network. @@ -375,7 +375,7 @@ def add_RPS_constraints(n, config, snakemake=None, sector=False): # noqa: ARG00 f"RPS demand basis for {constraint_row.region} ({constraint_row.planning_horizon}): " f"net_load={region_demand / 1e6:.1f} TWh, " f"btm_rooftop={rooftop_gen / 1e6:.2f} TWh, " - f"adjusted_rhs={region_rps_rhs / 1e6:.1f} TWh" + f"adjusted_rhs={region_rps_rhs / 1e6:.1f} TWh", ) # Iterate through constraints and add RPS constraints to the model diff --git a/workflow/scripts/retrieve_small_scale_solar.py b/workflow/scripts/retrieve_small_scale_solar.py index 5515854e6..dc5a35956 100644 --- a/workflow/scripts/retrieve_small_scale_solar.py +++ b/workflow/scripts/retrieve_small_scale_solar.py @@ -50,12 +50,54 @@ # All CONUS state abbreviations covered by pypsa-usa _STATES = [ - "AL", "AZ", "AR", "CA", "CO", "CT", "DE", "FL", "GA", - "ID", "IL", "IN", "IA", "KS", "KY", "LA", "ME", "MD", - "MA", "MI", "MN", "MS", "MO", "MT", "NE", "NV", "NH", - "NJ", "NM", "NY", "NC", "ND", "OH", "OK", "OR", "PA", - "RI", "SC", "SD", "TN", "TX", "UT", "VT", "VA", "WA", - "WV", "WI", "WY", + "AL", + "AZ", + "AR", + "CA", + "CO", + "CT", + "DE", + "FL", + "GA", + "ID", + "IL", + "IN", + "IA", + "KS", + "KY", + "LA", + "ME", + "MD", + "MA", + "MI", + "MN", + "MS", + "MO", + "MT", + "NE", + "NV", + "NH", + "NJ", + "NM", + "NY", + "NC", + "ND", + "OH", + "OK", + "OR", + "PA", + "RI", + "SC", + "SD", + "TN", + "TX", + "UT", + "VT", + "VA", + "WA", + "WV", + "WI", + "WY", ] @@ -97,8 +139,7 @@ def _fetch_from_eia(api_key: str, start_year: int, end_year: int) -> pd.DataFram if not records: raise ValueError( - "EIA API returned no small-scale solar data. " - "Check your API key and the date range." + "EIA API returned no small-scale solar data. Check your API key and the date range.", ) df = pd.DataFrame(records) @@ -114,8 +155,7 @@ def _fetch_from_eia(api_key: str, start_year: int, end_year: int) -> pd.DataFram df = df.sort_values(["state", "year"]).reset_index(drop=True) logger.info( - f"Retrieved {len(df)} state-year observations of small-scale solar " - f"from EIA API ({start_year}–{end_year})." + f"Retrieved {len(df)} state-year observations of small-scale solar from EIA API ({start_year}–{end_year}).", ) return df @@ -125,7 +165,7 @@ def _load_fallback(fallback_path: str) -> pd.DataFrame: logger.warning( "No EIA API key provided. Loading bundled small-scale solar data from " f"{fallback_path}. This data may not match your planning horizons exactly; " - "the most recent available year will be used for future years." + "the most recent available year will be used for future years.", ) df = pd.read_csv(fallback_path, dtype={"state": str, "year": int, "generation_mwh": float}) return df @@ -170,6 +210,5 @@ def _load_fallback(fallback_path: str) -> pd.DataFrame: df.to_csv(output_path, index=False) logger.info( - f"Small-scale solar data written to {output_path} " - f"({len(df)} state-year rows)." + f"Small-scale solar data written to {output_path} ({len(df)} state-year rows).", ) diff --git a/workflow/scripts/test/test_policy.py b/workflow/scripts/test/test_policy.py index 8668d2006..a9dea7091 100644 --- a/workflow/scripts/test/test_policy.py +++ b/workflow/scripts/test/test_policy.py @@ -384,9 +384,7 @@ def extra_functionality(n, _): # Verify that rooftop solar generation is counted on the LHS of the constraint region_buses = get_region_buses(n, ["CA"]) - rooftop_gens = n.generators[ - (n.generators.bus.isin(region_buses.index)) & (n.generators.carrier == "solar-rooftop") - ] + rooftop_gens = n.generators[(n.generators.bus.isin(region_buses.index)) & (n.generators.carrier == "solar-rooftop")] assert not rooftop_gens.empty, "Rooftop solar generator missing from CA region" rooftop_gen = n.generators_t.p[rooftop_gens.index].sum().sum() @@ -457,7 +455,7 @@ def _eligible_gen(n): logger.info( f"BTM credit test: CA eligible gen without BTM = {gen_no_btm:.1f} MWh, " - f"with BTM = {gen_with_btm:.1f} MWh (reduction = {gen_no_btm - gen_with_btm:.1f} MWh)" + f"with BTM = {gen_with_btm:.1f} MWh (reduction = {gen_no_btm - gen_with_btm:.1f} MWh)", ) assert gen_with_btm <= gen_no_btm + 1e-3, ( From 58b59498228e6b78921bdd72c1b9df2ef307a2e4 Mon Sep 17 00:00:00 2001 From: Wilson McNeil Date: Fri, 14 Aug 2026 11:00:37 -0700 Subject: [PATCH 6/7] refactor(rps): consolidate EIA fetch into eia module, clarify BTM demand scope Per review feedback: - Add SmallScaleSolar / _SmallScaleSolarData to eia.py, following the existing EiaData / DataExtractor factory pattern used by all other EIA data classes (ElectricPowerData, EnergyDemand, etc.) - Simplify retrieve_small_scale_solar.py to delegate API calls to the new class; removes duplicated requests logic and the hardcoded _STATES list (state filtering is now handled in _SmallScaleSolarData.format_data, consistent with _ElectricPowerOperationalData) - Expand add_RPS_constraints docstring to explain that the BTM solar credit is only appropriate for EIA 930 net-load profiles (demand.profile: eia) and should be disabled for EFS / AEO gross-load projections Co-Authored-By: Claude Sonnet 4.6 --- workflow/scripts/eia.py | 83 ++++++++++++ workflow/scripts/opts/policy.py | 10 ++ .../scripts/retrieve_small_scale_solar.py | 127 +----------------- 3 files changed, 99 insertions(+), 121 deletions(-) diff --git a/workflow/scripts/eia.py b/workflow/scripts/eia.py index 5d27c9145..a44a9f4db 100644 --- a/workflow/scripts/eia.py +++ b/workflow/scripts/eia.py @@ -490,6 +490,43 @@ def data_creator(self): return _ElectricPowerOperationalData(self.sector, self.year, self.api_key) +class SmallScaleSolar(EiaData): + """State-level small-scale (behind-the-meter) solar PV annual generation. + + Fetches sector-98 (below 1 MW nameplate) solar generation from the EIA + Electric Power Operational Data API for a range of years. + + Parameters + ---------- + start_year : int + First year to retrieve (EIA data available from ~2014). + end_year : int + Last year to retrieve. Capped at the latest available year. + api : str + EIA API v2 key. + + Examples + -------- + >>> df = SmallScaleSolar(2014, 2023, "YOUR_API_KEY").get_data() + """ + + LAST_AVAILABLE_YEAR: ClassVar[int] = 2023 + + def __init__(self, start_year: int, end_year: int, api: str) -> None: + self.start_year = start_year + self.end_year = min(end_year, self.LAST_AVAILABLE_YEAR) + self.api = api + if end_year > self.LAST_AVAILABLE_YEAR: + logger.warning( + f"Small-scale solar data only available through {self.LAST_AVAILABLE_YEAR}. " + f"Capping end_year at {self.LAST_AVAILABLE_YEAR}.", + ) + + def data_creator(self): + """Initializes data extractor.""" + return _SmallScaleSolarData(self.start_year, self.end_year, self.api) + + # product class DataExtractor(ABC): """Extracts and formats data.""" @@ -1839,6 +1876,52 @@ def format_data(self, df: pd.DataFrame) -> pd.DataFrame: return self._assign_dtypes(df) +class _SmallScaleSolarData(DataExtractor): + """Fetches annual small-scale (behind-the-meter) solar generation by state. + + Uses the EIA Electric Power Operational Data endpoint with sector 98 + (small-scale photovoltaic, below 1 MW nameplate) and fuel type SUN. + Generation values are returned in MWh (converted from the API's thousand MWh). + """ + + SECTOR_ID: ClassVar[int] = 98 + FUEL_TYPE: ClassVar[str] = "SUN" + + def __init__(self, start_year: int, end_year: int, api_key: str) -> None: + super().__init__(end_year, api_key) # self.year = end_year + self.start_year = start_year + + def build_url(self) -> str: + base_url = "electricity/electric-power-operational-data/data/" + facets = ( + f"frequency=annual&data[0]=generation" + f"&facets[fueltypeid][]={self.FUEL_TYPE}" + f"&facets[sectorid][]={self.SECTOR_ID}" + f"&start={self.start_year}&end={self.year}" + f"&sort[0][column]=period&sort[0][direction]=asc" + f"&offset=0&length=5000" + ) + return f"{API_BASE}{base_url}?api_key={self.api_key}&{facets}" + + def format_data(self, df: pd.DataFrame) -> pd.DataFrame: + df = df.rename( + columns={ + "location": "state", + "period": "year", + "generation": "generation_mwh", + }, + ) + df = df[["state", "year", "generation_mwh"]].copy() + df["year"] = df["year"].astype(int) + # EIA reports generation in thousand MWh; convert to MWh + df["generation_mwh"] = pd.to_numeric(df["generation_mwh"], errors="coerce") * 1_000 + df = df.dropna(subset=["generation_mwh"]) + df = df[df["generation_mwh"] > 0] + # Exclude national aggregates (e.g., "US") and sub-state numeric codes + df = df[(df["state"].str.len() == 2) & (df["state"] != "US")] + return df.sort_values(["state", "year"]).reset_index(drop=True) + + if __name__ == "__main__": with open("./../config/config.api.yaml") as file: yaml_data = yaml.safe_load(file) diff --git a/workflow/scripts/opts/policy.py b/workflow/scripts/opts/policy.py index 239156af2..4cb3e7236 100644 --- a/workflow/scripts/opts/policy.py +++ b/workflow/scripts/opts/policy.py @@ -286,6 +286,16 @@ def add_RPS_constraints(n, config, snakemake=None, sector=False): rhs = pct * net_load - (1 - pct) * rooftop_gen = pct * gross_load - rooftop_gen + **Demand source compatibility:** The BTM credit is only appropriate when + the network load time-series is derived from EIA 930 *net* generation data + (``demand.profile: eia``), in which case behind-the-meter solar is already + subtracted from the reported load. If the demand profile is sourced from + EFS or AEO projections — which typically report *gross* electricity sales + and do not subtract BTM generation — the BTM credit should not be applied. + To disable it, simply omit the ``small_scale_solar`` input from the + ``solve_network`` rule (or leave ``api.eia`` unconfigured so the fallback + CSV is not forward-filled beyond its data year). + Parameters ---------- n : pypsa.Network diff --git a/workflow/scripts/retrieve_small_scale_solar.py b/workflow/scripts/retrieve_small_scale_solar.py index dc5a35956..0aa5a2e5f 100644 --- a/workflow/scripts/retrieve_small_scale_solar.py +++ b/workflow/scripts/retrieve_small_scale_solar.py @@ -15,7 +15,6 @@ - ``fueltypeid = SUN`` - ``sectorid = 98`` (small-scale, i.e. below 1 MW threshold) - ``frequency = annual`` - - ``facets[location]`` = US state abbreviations **Outputs** @@ -37,128 +36,10 @@ from pathlib import Path import pandas as pd -import requests +from eia import SmallScaleSolar logger = logging.getLogger(__name__) -# EIA API v2 endpoint for electric power operational data -_EIA_URL = "https://api.eia.gov/v2/electricity/electric-power-operational-data/data/" - -# Sector 98 = small-scale solar (below 1 MW nameplate) -_SECTOR_ID = "98" -_FUEL_TYPE = "SUN" - -# All CONUS state abbreviations covered by pypsa-usa -_STATES = [ - "AL", - "AZ", - "AR", - "CA", - "CO", - "CT", - "DE", - "FL", - "GA", - "ID", - "IL", - "IN", - "IA", - "KS", - "KY", - "LA", - "ME", - "MD", - "MA", - "MI", - "MN", - "MS", - "MO", - "MT", - "NE", - "NV", - "NH", - "NJ", - "NM", - "NY", - "NC", - "ND", - "OH", - "OK", - "OR", - "PA", - "RI", - "SC", - "SD", - "TN", - "TX", - "UT", - "VT", - "VA", - "WA", - "WV", - "WI", - "WY", -] - - -def _fetch_from_eia(api_key: str, start_year: int, end_year: int) -> pd.DataFrame: - """Download small-scale solar annual generation from the EIA API v2.""" - records = [] - offset = 0 - page_size = 5000 - - while True: - params = { - "api_key": api_key, - "frequency": "annual", - "data[0]": "generation", - "facets[fueltypeid][]": _FUEL_TYPE, - "facets[sectorid][]": _SECTOR_ID, - "start": str(start_year), - "end": str(end_year), - "sort[0][column]": "period", - "sort[0][direction]": "asc", - "offset": offset, - "length": page_size, - } - - response = requests.get(_EIA_URL, params=params, timeout=60) - response.raise_for_status() - payload = response.json() - - data = payload.get("response", {}).get("data", []) - if not data: - break - - records.extend(data) - - total = payload.get("response", {}).get("total", 0) - offset += page_size - if offset >= int(total): - break - - if not records: - raise ValueError( - "EIA API returned no small-scale solar data. Check your API key and the date range.", - ) - - df = pd.DataFrame(records) - # API returns location as state abbreviation, period as "YYYY" - df = df.rename(columns={"location": "state", "period": "year", "generation": "generation_mwh"}) - df = df[["state", "year", "generation_mwh"]].copy() - df["year"] = df["year"].astype(int) - # EIA reports generation in thousand MWh; convert to MWh - df["generation_mwh"] = pd.to_numeric(df["generation_mwh"], errors="coerce") * 1_000 - df = df.dropna(subset=["generation_mwh"]) - df = df[df["generation_mwh"] > 0] - df = df[df["state"].isin(_STATES)] - df = df.sort_values(["state", "year"]).reset_index(drop=True) - - logger.info( - f"Retrieved {len(df)} state-year observations of small-scale solar from EIA API ({start_year}–{end_year}).", - ) - return df - def _load_fallback(fallback_path: str) -> pd.DataFrame: """Load a pre-bundled CSV when no EIA API key is available.""" @@ -188,7 +69,11 @@ def _load_fallback(fallback_path: str) -> pd.DataFrame: # EIA small-scale solar data starts around 2014. start_year = 2014 end_year = max(planning_horizons) - df = _fetch_from_eia(api_key, start_year, end_year) + df = SmallScaleSolar(start_year, end_year, api_key).get_data() + logger.info( + f"Retrieved {len(df)} state-year observations of small-scale solar " + f"from EIA API ({start_year}–{end_year})." + ) else: df = _load_fallback(fallback_path) From ebd6e96dfc4337a1e7f9226a0d2af193bfe35343 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 14 Aug 2026 18:02:56 +0000 Subject: [PATCH 7/7] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- workflow/scripts/retrieve_small_scale_solar.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/workflow/scripts/retrieve_small_scale_solar.py b/workflow/scripts/retrieve_small_scale_solar.py index 0aa5a2e5f..f56007819 100644 --- a/workflow/scripts/retrieve_small_scale_solar.py +++ b/workflow/scripts/retrieve_small_scale_solar.py @@ -71,8 +71,7 @@ def _load_fallback(fallback_path: str) -> pd.DataFrame: end_year = max(planning_horizons) df = SmallScaleSolar(start_year, end_year, api_key).get_data() logger.info( - f"Retrieved {len(df)} state-year observations of small-scale solar " - f"from EIA API ({start_year}–{end_year})." + f"Retrieved {len(df)} state-year observations of small-scale solar from EIA API ({start_year}–{end_year}).", ) else: df = _load_fallback(fallback_path)