data update 2026

powerplantmatching/data.py

@@ -1612,17 +1612,22 @@ def IRENASTAT(raw=False, update=False, config=None):
         return df
 
     RENAME_COLUMNS = {
-        "Electricity statistics": "Capacity",
+        "Electricity capacity statistics": "Capacity",
         "Country/area": "Country",
         "Grid connection": "Grid",
     }
     df.rename(columns=RENAME_COLUMNS, inplace=True)
 
-    df.drop(columns="Data Type", inplace=True)
-
-    # Rename all entries "Congo (the)" to "Congo" under the column
-    # "Country"; the former confuses country_converter.
-    df["Country"] = df["Country"].replace("Congo (the)", "Congo")
+    # Rename country entries that confuse country_converter
+    country_renames = {
+        "Congo (the)": "Congo",
+        "Fr Polynesia": "French Polynesia",
+        "Amer Samoa": "American Samoa",
+        "Cent Afr Rep": "Central African Republic",
+        "St Pierre Mq": "Saint Pierre and Miquelon",
+        "New Caledon": "New Caledonia",
+    }
+    df["Country"] = df["Country"].replace(country_renames)
 
     # Consistent country names for dataset
     df = convert_to_short_name(df)
@@ -1631,7 +1636,8 @@ def IRENASTAT(raw=False, update=False, config=None):
 
     # Remove all rows where Technology is just a Total
     df = df[
-        ~df.Technology.str.contains("Total Renewable|Total Non-Renewable", na=False)
+        ~df.Technology.str.startswith("Total", na=False)
+        & ~df.Technology.str.contains("Solar energy|Wind energy|Bioenergy", na=False)
     ]
 
     fueltype_dict = {
@@ -1640,30 +1646,30 @@ def IRENASTAT(raw=False, update=False, config=None):
         "Onshore wind energy": "Wind",
         "Offshore wind energy": "Wind",
         "Renewable hydropower": "Hydro",
-        "Mixed Hydro Plants": "Hydro",
-        "Pumped storage": "Hydro",
+        "Mixed hydropower": "Hydro",
+        "Pumped hydro": "Hydro",
         "Solid biofuels": "Solid Biomass",
-        "Renewable municipal waste": "Waste",
+        "Renewable waste": "Waste",
+        "Non-renewable waste": "Waste",
         "Liquid biofuels": "Solid Biomass",
-        "Biogas": "Biogas",
+        "Gas biofuels": "Biogas",
         "Geothermal energy": "Geothermal",
         "Marine energy": "Marine",
-        "Coal and peat": "Hard Coal",
+        "Coal": "Hard Coal",
         "Oil": "Oil",
         "Natural gas": "Natural Gas",
-        "Nuclear": "Nuclear",
-        "Fossil fuels n.e.s.": "Other",
+        "Nuclear energy": "Nuclear",
+        "Fossil fuels": "Other",
         "Other non-renewable energy": "Other",
+        "Other non-renewable energy n.e.s.": "Other",
     }
 
     technology_dict = {
         "Solar photovoltaic": "PV",
         "Solar thermal energy": "CSP",
         "Onshore wind energy": "Onshore",
         "Offshore wind energy": "Offshore",
-        "Pumped storage": "Pumped Storage",
-        "Geothermal energy": "Geothermal",
-        "Marine energy": "Marine",
+        "Pumped hydro": "Pumped Storage",
     }
 
     df["Fueltype"] = df.Technology.map(fueltype_dict)
@@ -2324,6 +2330,19 @@ def MASTR(
     Provided by the German Federal Network Agency (Bundesnetzagentur / BNetzA) and
     contains data on Germany, Austria and Switzerland.
 
+    To retrieve an up-to-date version, run:
+
+    ```python
+    # uv add open-mastr
+    from open_mastr import Mastr
+    db = Mastr()
+    db.download()
+    db.to_csv()
+    ```
+
+    This will store the data in `~/.open-MaStR/data/dataversion-YYYY-MM-DD`.
+    Link to a zipped version of this folder in `config.yaml`.
+
     Parameters
     ----------
     raw : Boolean, default False
@@ -2339,7 +2358,7 @@ def MASTR(
 
     config = get_config() if config is None else config
 
-    THRESHOLD_KW = config["MASTR"].get("capacity_threshold", 0.1) * 1e3  # noqa: F841
+    THRESHOLD_KW = config["MASTR"].get("capacity_threshold", 0.05) * 1e3  # noqa: F841
 
     RENAME_COLUMNS = {
         "EinheitMastrNummer": "projectID",
@@ -2369,6 +2388,29 @@ def MASTR(
         "NameWindpark",
         "Technologie",
     ]
+    # columns whose type inference is ambiguous, either between chunks or files
+    STR_COLUMNS = pd.Index(
+        PARSE_COLUMNS
+        + [
+            "Batterietechnologie",
+            "DatumBeginnVoruebergehendeStilllegung",
+            "DatumEndgueltigeStilllegung",
+            "DatumWiederaufnahmeBetrieb",
+            "EinheitBetriebsstatus",
+            "EinheitMastrNummer",
+            "Gemeinde",
+            "GeplantesInbetriebnahmedatum",
+            "Inbetriebnahmedatum",
+            "KwkMastrNummer",
+            "Land",
+            "Landkreis",
+            "WindAnLandOderAufSee",
+            "NameKraftwerk",
+            "Ort",
+            "Postleitzahl",
+            "WEIC",
+        ]
+    )
 
     fn = get_raw_file("MASTR", update=update, config=config)
     file_suffixes = {
@@ -2378,7 +2420,7 @@ def MASTR(
         "Hydro": "hydro_raw.csv",
         "Wind": "wind_raw.csv",
         "Solar": "solar_raw.csv",
-        "Storage": "bnetza_mastr_storage_raw.csv",
+        "Storage": "storage_raw.csv",
     }
     data_frames = []
     with ZipFile(fn, "r") as file:
@@ -2397,27 +2439,31 @@ def MASTR(
                         "Ort",
                         "Gemeinde",
                         "Landkreis",
-                        "Lage",
+                        "WindAnLandOderAufSee",
                     ]
                     target_columns = (
                         target_columns + PARSE_COLUMNS + list(RENAME_COLUMNS.keys())
                     )
                     usecols = available_columns.intersection(target_columns)
-                    df = (
-                        pd.read_csv(file.open(name), usecols=usecols, low_memory=False)
-                        .assign(Filesuffix=fueltype)
-                        .query("Nettonennleistung >= @THRESHOLD_KW")
+                    dtypes = {c: "str" for c in usecols.intersection(STR_COLUMNS)}
+                    chunks = pd.read_csv(
+                        file.open(name),
+                        usecols=usecols,
+                        dtype=dtypes,
+                        chunksize=100_000,
                     )
+                    df = pd.concat(
+                        [c.query("Nettonennleistung >= @THRESHOLD_KW") for c in chunks]
+                    ).assign(Filesuffix=fueltype)
                     data_frames.append(df)
                     break
     df = pd.concat(data_frames).reset_index(drop=True)
 
     cols = ["NutzbareSpeicherkapazitaet", "VerknuepfteEinheit"]
     with ZipFile(fn, "r") as file:
-        fn_storage_units = (
-            "bnetza_open_mastr_2025-02-09/bnetza_mastr_storage_units_raw.csv"
-        )
-        storage_units = pd.read_csv(file.open(fn_storage_units), usecols=cols)
+        for name in file.namelist():
+            if name.endswith("storage_units_raw.csv"):
+                storage_units = pd.read_csv(file.open(name), usecols=cols)
 
     storage_mwh = (
         storage_units.assign(
@@ -2514,9 +2560,9 @@ def MASTR(
         "Windkraft an Land": "Onshore",
     }
     wind = df_processed.query("Energietraeger == 'Wind'").index
-    df_processed.loc[wind, "Technology"] = df_processed.loc[wind, "Lage"].map(
-        WIND_MAPPING
-    )
+    df_processed.loc[wind, "Technology"] = df_processed.loc[
+        wind, "WindAnLandOderAufSee"
+    ].map(WIND_MAPPING)
 
     sel = df_processed.query(
         "Fueltype == 'Natural Gas' and Filesuffix == 'Bioenergy'"
@@ -2631,18 +2677,18 @@ def EESI(
     if raw:
         return df
 
-    status_list = config["EESI"].get("status", ["Operational"])  # noqa: F841
+    status_list = config["EESI"].get("status_name", ["Operational"])  # noqa: F841
 
     RENAME_COLUMNS = {
         "title": "Name",
         "power": "Capacity",
         "capacity": "StorageCapacity_MWh",
         "facility_latitude": "lat",
         "facility_longitude": "lon",
-        "facility_country": "Country",
+        "facility_country_name": "Country",
         "id": "projectID",
         "technology_name": "Technology",
-        "status": "Status",
+        "status_name": "Status",
     }
 
     df_processed = (
@@ -2658,7 +2704,7 @@ def EESI(
         )
     )
 
-    sel = df_processed.query("technology_parentName == 'ElectroChemical'").index
+    sel = df_processed.query("technology_parentName == 'Electrochemical'").index
     df_processed.loc[sel, "Fueltype"] = "Battery"
 
     sel = df_processed.query("technology_parentName == 'Thermal'").index
@@ -2678,14 +2724,22 @@ def EESI(
         "Lithium-ion batteries": "Li",
         "Lead Acid batteries": "Pb",
         "Sodium Sulphur batteries": "NaS",
+        "Lithium iron phosphate battery (LFP)": "Li",
+        "Lithium nickel manganese cobalt oxide battery (NMC)": "Li",
+        "Lithium nickel cobalt aluminium oxide battery (NCA)": "Li",
+        "Lithium manganese oxide battery (LMO)": "Li",
+        "Lithium-titanate battery (LTO)": "Li",
+        "Lithium-Metal-Polymer batteries": "Li",
         "Redox flow batteries Vanadium": "V",
         "Sodium Nickel Chloride batteries": "NaNiCl",
-        "Lithium-titanate battery (LTO)": "Li",
         "Pumped Hydro Storage (PHS)": "Pumped Storage",
         "Unespecified Storage - mechanical": np.nan,
         "Compressed Air Energy Storage (CAES)": "CAES",
-        "Flywheel Energy Storage": "Flywheel",
+        "Liquid Air Energy Storage (LAES)": "LAES",
+        "Iron air battery": "Fe",
+        "Flywheel": "Flywheel",
         "Unspecific Thermal Storage": np.nan,
+        "Unspecific Sensible Thermal Energy Storage (STES)": np.nan,
         "Molten salts (Sensible Thermal Energy Storage (STES))": "Molten Salt",
     }
     df_processed.Technology = df_processed.Technology.map(TECHNOLOGY_MAPPING)

powerplantmatching/package_data/config.yaml

@@ -72,30 +72,30 @@ BEYONDCOAL:
   aggregated_units: false
   reliability_score: 4
   status: ["construction", "operational", "no longer coal", "retired", "standby", "deactivated", "retrofitting"]
-  fn: 2025-07-24-BeyondFossilFuels-Europe_Coal_Plants_Database.xlsx
-  url: https://tubcloud.tu-berlin.de/s/bw7tbpo6A3qXYDz/download/2025-07-24-BeyondFossilFuels-Europe_Coal_Plants_Database.xlsx
+  fn:  20260731-BeyondFossilFuels-Europe_Coal_Plants_Database.xlsx
+  url: https://tubcloud.tu-berlin.de/s/qHfNEELYy6EnBSX/download/20260731-BeyondFossilFuels-Europe_Coal_Plants_Database.xlsx
 IRENA:
   net_capacity: true
   aggregated_units: true
-  fn: IRENASTAT_capacities_2000-2024.csv
-  # compiled from https://pxweb.irena.org/pxweb/en/IRENASTAT/IRENASTAT__Power%20Capacity%20and%20Generation/Country_ELECSTAT_2025_H2_PX.px/
-  url: https://tubcloud.tu-berlin.de/s/dDS9erreKPNH4Ey/download/IRENASTAT_capacities_2000-2024.csv
+  fn: IRENASTAT_capacities_2000-2025.csv
+  # compiled from https://pxweb.irena.org/pxweb/en/IRENASTAT/IRENASTAT__Power%20Capacity%20and%20Generation/Country_ELECCAP_2026_H1_v-PX%201.px/
+  url: https://tubcloud.tu-berlin.de/s/qALFkR8eB73FWA9/download/IRENASTAT_capacities_2000-2025.csv
 CARMA:
   net_capacity: false
   reliability_score: 1
   url: https://raw.githubusercontent.com/pypsa-meets-earth/ppm-data-backup/main/Full_CARMA_2009_Dataset_1.csv
   fn: Full_CARMA_2009_Dataset_1.csv
 ENTSOE:
   reliability_score: 5
-  url: https://tubcloud.tu-berlin.de/s/N7qo3AGyRYZyisS/download/entsoe_transparency_platform_20250820.csv
-  fn: entsoe_transparency_platform_20250820.csv
+  url: https://tubcloud.tu-berlin.de/s/MfA7aHLKDZzNkeW/download/entsoe_transparency_platform_20260816-1122.csv
+  fn: entsoe_transparency_platform_20260816-1122.csv
 ENTSOE-EIC:
   url: https://eepublicdownloads.blob.core.windows.net/cio-lio/csv/W_eicCodes.csv
   fn: W_eicCodes.csv
 JRC:
   reliability_score: 5
   fn: jrc-hydro-power-plant-database.csv
-  url: https://raw.githubusercontent.com/energy-modelling-toolkit/hydro-power-database/27e80f/data/jrc-hydro-power-plant-database.csv
+  url: https://raw.githubusercontent.com/energy-modelling-toolkit/hydro-power-database/3e8a837/data/jrc-hydro-power-plant-database.csv
 GEO:
   net_capacity: false
   reliability_score: 2
@@ -172,8 +172,8 @@ GGPT:
   net_capacity: false
   reliability_score: 6
   status: ["operating", "retired", "construction"]
-  fn: Global-Oil-and-Gas-Plant-Tracker-GOGPT-August-2025.xlsx
-  url: https://tubcloud.tu-berlin.de/s/WrmNX5awNJFcXrQ/download/Global-Oil-and-Gas-Plant-Tracker-GOGPT-August-2025.xlsx
+  fn: Global-Oil-and-Gas-Plant-Tracker-GOGPT-January-2026.xlsx
+  url: https://tubcloud.tu-berlin.de/s/oTNEPzKcTZjdEZW/download/Global-Oil-and-Gas-Plant-Tracker-GOGPT-January-2026.xlsx
 GEM:
   # combined data set of all GEM trackers
   net_capacity: true
@@ -182,15 +182,15 @@ GCPT:
   net_capacity: false
   reliability_score: 6
   status: ["operating", "retired", "construction", "mothballed"]
-  fn: Global-Coal-Plant-Tracker-July-2025.xlsx
-  url: https://tubcloud.tu-berlin.de/s/ijzbscopNTgNB2r/download/Global-Coal-Plant-Tracker-July-2025.xlsx
+  fn: Global-Coal-Plant-Tracker-July-2026.xlsx
+  url: https://tubcloud.tu-berlin.de/s/xyZSXcd4QNSzDsy/download/Global-Coal-Plant-Tracker-July-2026.xlsx
 GGTPT:
   net_capacity: false
   reliability_score: 6
   aggregated_units: false
   status: ["operating", "retired", "construction", "mothballed"]
-  fn: Geothermal-Power-Tracker-March-2025-Final.xlsx
-  url: https://tubcloud.tu-berlin.de/s/ypr3eL2K5kckAK4/download/Geothermal-Power-Tracker-March-2025-Final.xlsx
+  fn: Geothermal-Power-Tracker-March-2026-Final.xlsx
+  url: https://tubcloud.tu-berlin.de/s/spGtyE3oPZrMeB3/download/Geothermal-Power-Tracker-March-2026-Final.xlsx
 GWPT:
   net_capacity: false
   reliability_score: 6
@@ -213,27 +213,28 @@ GNPT:
   net_capacity: false
   reliability_score: 6
   status: ["operating", "retired", "mothballed", "construction"]
-  fn: Global-Nuclear-Power-Tracker-July-2024.xlsx
-  url: https://tubcloud.tu-berlin.de/s/gXFim9EciRHrjeQ/download/Global-Nuclear-Power-Tracker-July-2024.xlsx
+  fn: Global-Nuclear-Power-Tracker-September-2025.xlsx
+  url: https://tubcloud.tu-berlin.de/s/NBX6G2eyZ3T97gq/download/Global-Nuclear-Power-Tracker-September-2025.xlsx
 GHPT:
   net_capacity: false
   reliability_score: 6
   status: ["operating", "retired", "construction"]
-  fn: Global-Hydropower-Tracker-April-2025.xlsx
-  url: https://tubcloud.tu-berlin.de/s/aDyd3MJWZNgeEH4/download/Global-Hydropower-Tracker-April-2025.xlsx
+  fn: Global-Hydropower-Tracker-March-2026.xlsx
+  url: https://tubcloud.tu-berlin.de/s/feKWmAfzPtHPHN2/download/Global-Hydropower-Tracker-March-2026.xlsx
 MASTR:
   net_capacity: true
   reliability_score: 7
   status: ["In Betrieb", "In Planung", "Endgültig stillgelegt", "Vorübergehend stillgelegt"]
-  capacity_threshold: 0.1  # all values below will be filtered out, given in MW
-  fn: bnetza_open_mastr_2025-02-09.zip
-  url: https://zenodo.org/records/14783581/files/bnetza_open_mastr_2025-02-09.zip
+  capacity_threshold: 0.05  # all values below will be filtered out, given in MW
+  fn: dataversion-2026-08-16.zip
+  url: https://tubcloud.tu-berlin.de/s/KJWoc8rwezoRkcQ/download/dataversion-2026-08-16.zip
 EESI:
   net_capacity: true
   reliability_score: 5
   status: ["Operational"] # since no start years given
-  fn: european-energy-storage-inventory-20250817-2245.json
-  url: https://tubcloud.tu-berlin.de/s/5KqMDMZfb2pN3Aw/download/european-energy-storage-inventory-20250817-2245.json
+  fn: european-energy-storage-inventory-20260816-1026.json
+  # from https://ses.jrc.ec.europa.eu/storage-inventory-tool/api/projects
+  url: https://tubcloud.tu-berlin.de/s/ijiN2EWjHfg4NXf/download/european-energy-storage-inventory-20260816-1026.json
 GND:
   net_capacity: true
   reliability_score: 5

powerplantmatching/utils.py

@@ -9,6 +9,7 @@
 import multiprocessing
 import os
 import re
+import shutil
 from ast import literal_eval as liteval
 from importlib.metadata import version
 
@@ -80,13 +81,17 @@ def get_raw_file(name, update=False, config=None, skip_retrieve=False):
 
     if (not os.path.exists(path) or update) and not skip_retrieve:
         url = df_config["url"]
-        logger.info(f"Retrieving data from {url}")
-        base_version = parse(version(__package__)).base_version
-        user_agent = f"{__package__}/{base_version}"
-        r = requests.get(url, headers={"User-Agent": user_agent}, timeout=60)
-        r.raise_for_status()
-        with open(path, "wb") as outfile:
-            outfile.write(r.content)
+        if os.path.exists(url):
+            logger.info(f"Copying data from local file {url}")
+            shutil.copyfile(url, path)
+        else:
+            logger.info(f"Retrieving data from {url}")
+            base_version = parse(version(__package__)).base_version
+            user_agent = f"{__package__}/{base_version}"
+            r = requests.get(url, headers={"User-Agent": user_agent}, timeout=60)
+            r.raise_for_status()
+            with open(path, "wb") as outfile:
+                outfile.write(r.content)
 
     return path