diff --git a/powerplantmatching/data.py b/powerplantmatching/data.py index 08fd526..08fbf73 100644 --- a/powerplantmatching/data.py +++ b/powerplantmatching/data.py @@ -1612,17 +1612,22 @@ def IRENASTAT(raw=False, update=False, config=None): return df RENAME_COLUMNS = { - "Electricity statistics": "Capacity", + "Electricity capacity statistics": "Capacity", "Country/area": "Country", "Grid connection": "Grid", } df.rename(columns=RENAME_COLUMNS, inplace=True) - df.drop(columns="Data Type", inplace=True) - - # Rename all entries "Congo (the)" to "Congo" under the column - # "Country"; the former confuses country_converter. - df["Country"] = df["Country"].replace("Congo (the)", "Congo") + # Rename country entries that confuse country_converter + country_renames = { + "Congo (the)": "Congo", + "Fr Polynesia": "French Polynesia", + "Amer Samoa": "American Samoa", + "Cent Afr Rep": "Central African Republic", + "St Pierre Mq": "Saint Pierre and Miquelon", + "New Caledon": "New Caledonia", + } + df["Country"] = df["Country"].replace(country_renames) # Consistent country names for dataset df = convert_to_short_name(df) @@ -1631,7 +1636,8 @@ def IRENASTAT(raw=False, update=False, config=None): # Remove all rows where Technology is just a Total df = df[ - ~df.Technology.str.contains("Total Renewable|Total Non-Renewable", na=False) + ~df.Technology.str.startswith("Total", na=False) + & ~df.Technology.str.contains("Solar energy|Wind energy|Bioenergy", na=False) ] fueltype_dict = { @@ -1640,20 +1646,22 @@ def IRENASTAT(raw=False, update=False, config=None): "Onshore wind energy": "Wind", "Offshore wind energy": "Wind", "Renewable hydropower": "Hydro", - "Mixed Hydro Plants": "Hydro", - "Pumped storage": "Hydro", + "Mixed hydropower": "Hydro", + "Pumped hydro": "Hydro", "Solid biofuels": "Solid Biomass", - "Renewable municipal waste": "Waste", + "Renewable waste": "Waste", + "Non-renewable waste": "Waste", "Liquid biofuels": "Solid Biomass", - "Biogas": "Biogas", + "Gas biofuels": "Biogas", "Geothermal energy": "Geothermal", "Marine energy": "Marine", - "Coal and peat": "Hard Coal", + "Coal": "Hard Coal", "Oil": "Oil", "Natural gas": "Natural Gas", - "Nuclear": "Nuclear", - "Fossil fuels n.e.s.": "Other", + "Nuclear energy": "Nuclear", + "Fossil fuels": "Other", "Other non-renewable energy": "Other", + "Other non-renewable energy n.e.s.": "Other", } technology_dict = { @@ -1661,9 +1669,7 @@ def IRENASTAT(raw=False, update=False, config=None): "Solar thermal energy": "CSP", "Onshore wind energy": "Onshore", "Offshore wind energy": "Offshore", - "Pumped storage": "Pumped Storage", - "Geothermal energy": "Geothermal", - "Marine energy": "Marine", + "Pumped hydro": "Pumped Storage", } df["Fueltype"] = df.Technology.map(fueltype_dict) @@ -2324,6 +2330,19 @@ def MASTR( Provided by the German Federal Network Agency (Bundesnetzagentur / BNetzA) and contains data on Germany, Austria and Switzerland. + To retrieve an up-to-date version, run: + + ```python + # uv add open-mastr + from open_mastr import Mastr + db = Mastr() + db.download() + db.to_csv() + ``` + + This will store the data in `~/.open-MaStR/data/dataversion-YYYY-MM-DD`. + Link to a zipped version of this folder in `config.yaml`. + Parameters ---------- raw : Boolean, default False @@ -2339,7 +2358,7 @@ def MASTR( config = get_config() if config is None else config - THRESHOLD_KW = config["MASTR"].get("capacity_threshold", 0.1) * 1e3 # noqa: F841 + THRESHOLD_KW = config["MASTR"].get("capacity_threshold", 0.05) * 1e3 # noqa: F841 RENAME_COLUMNS = { "EinheitMastrNummer": "projectID", @@ -2369,6 +2388,29 @@ def MASTR( "NameWindpark", "Technologie", ] + # columns whose type inference is ambiguous, either between chunks or files + STR_COLUMNS = pd.Index( + PARSE_COLUMNS + + [ + "Batterietechnologie", + "DatumBeginnVoruebergehendeStilllegung", + "DatumEndgueltigeStilllegung", + "DatumWiederaufnahmeBetrieb", + "EinheitBetriebsstatus", + "EinheitMastrNummer", + "Gemeinde", + "GeplantesInbetriebnahmedatum", + "Inbetriebnahmedatum", + "KwkMastrNummer", + "Land", + "Landkreis", + "WindAnLandOderAufSee", + "NameKraftwerk", + "Ort", + "Postleitzahl", + "WEIC", + ] + ) fn = get_raw_file("MASTR", update=update, config=config) file_suffixes = { @@ -2378,7 +2420,7 @@ def MASTR( "Hydro": "hydro_raw.csv", "Wind": "wind_raw.csv", "Solar": "solar_raw.csv", - "Storage": "bnetza_mastr_storage_raw.csv", + "Storage": "storage_raw.csv", } data_frames = [] with ZipFile(fn, "r") as file: @@ -2397,27 +2439,31 @@ def MASTR( "Ort", "Gemeinde", "Landkreis", - "Lage", + "WindAnLandOderAufSee", ] target_columns = ( target_columns + PARSE_COLUMNS + list(RENAME_COLUMNS.keys()) ) usecols = available_columns.intersection(target_columns) - df = ( - pd.read_csv(file.open(name), usecols=usecols, low_memory=False) - .assign(Filesuffix=fueltype) - .query("Nettonennleistung >= @THRESHOLD_KW") + dtypes = {c: "str" for c in usecols.intersection(STR_COLUMNS)} + chunks = pd.read_csv( + file.open(name), + usecols=usecols, + dtype=dtypes, + chunksize=100_000, ) + df = pd.concat( + [c.query("Nettonennleistung >= @THRESHOLD_KW") for c in chunks] + ).assign(Filesuffix=fueltype) data_frames.append(df) break df = pd.concat(data_frames).reset_index(drop=True) cols = ["NutzbareSpeicherkapazitaet", "VerknuepfteEinheit"] with ZipFile(fn, "r") as file: - fn_storage_units = ( - "bnetza_open_mastr_2025-02-09/bnetza_mastr_storage_units_raw.csv" - ) - storage_units = pd.read_csv(file.open(fn_storage_units), usecols=cols) + for name in file.namelist(): + if name.endswith("storage_units_raw.csv"): + storage_units = pd.read_csv(file.open(name), usecols=cols) storage_mwh = ( storage_units.assign( @@ -2514,9 +2560,9 @@ def MASTR( "Windkraft an Land": "Onshore", } wind = df_processed.query("Energietraeger == 'Wind'").index - df_processed.loc[wind, "Technology"] = df_processed.loc[wind, "Lage"].map( - WIND_MAPPING - ) + df_processed.loc[wind, "Technology"] = df_processed.loc[ + wind, "WindAnLandOderAufSee" + ].map(WIND_MAPPING) sel = df_processed.query( "Fueltype == 'Natural Gas' and Filesuffix == 'Bioenergy'" @@ -2631,7 +2677,7 @@ def EESI( if raw: return df - status_list = config["EESI"].get("status", ["Operational"]) # noqa: F841 + status_list = config["EESI"].get("status_name", ["Operational"]) # noqa: F841 RENAME_COLUMNS = { "title": "Name", @@ -2639,10 +2685,10 @@ def EESI( "capacity": "StorageCapacity_MWh", "facility_latitude": "lat", "facility_longitude": "lon", - "facility_country": "Country", + "facility_country_name": "Country", "id": "projectID", "technology_name": "Technology", - "status": "Status", + "status_name": "Status", } df_processed = ( @@ -2658,7 +2704,7 @@ def EESI( ) ) - sel = df_processed.query("technology_parentName == 'ElectroChemical'").index + sel = df_processed.query("technology_parentName == 'Electrochemical'").index df_processed.loc[sel, "Fueltype"] = "Battery" sel = df_processed.query("technology_parentName == 'Thermal'").index @@ -2678,14 +2724,22 @@ def EESI( "Lithium-ion batteries": "Li", "Lead Acid batteries": "Pb", "Sodium Sulphur batteries": "NaS", + "Lithium iron phosphate battery (LFP)": "Li", + "Lithium nickel manganese cobalt oxide battery (NMC)": "Li", + "Lithium nickel cobalt aluminium oxide battery (NCA)": "Li", + "Lithium manganese oxide battery (LMO)": "Li", + "Lithium-titanate battery (LTO)": "Li", + "Lithium-Metal-Polymer batteries": "Li", "Redox flow batteries Vanadium": "V", "Sodium Nickel Chloride batteries": "NaNiCl", - "Lithium-titanate battery (LTO)": "Li", "Pumped Hydro Storage (PHS)": "Pumped Storage", "Unespecified Storage - mechanical": np.nan, "Compressed Air Energy Storage (CAES)": "CAES", - "Flywheel Energy Storage": "Flywheel", + "Liquid Air Energy Storage (LAES)": "LAES", + "Iron air battery": "Fe", + "Flywheel": "Flywheel", "Unspecific Thermal Storage": np.nan, + "Unspecific Sensible Thermal Energy Storage (STES)": np.nan, "Molten salts (Sensible Thermal Energy Storage (STES))": "Molten Salt", } df_processed.Technology = df_processed.Technology.map(TECHNOLOGY_MAPPING) diff --git a/powerplantmatching/package_data/config.yaml b/powerplantmatching/package_data/config.yaml index aced4f3..e579de7 100644 --- a/powerplantmatching/package_data/config.yaml +++ b/powerplantmatching/package_data/config.yaml @@ -72,14 +72,14 @@ BEYONDCOAL: aggregated_units: false reliability_score: 4 status: ["construction", "operational", "no longer coal", "retired", "standby", "deactivated", "retrofitting"] - fn: 2025-07-24-BeyondFossilFuels-Europe_Coal_Plants_Database.xlsx - url: https://tubcloud.tu-berlin.de/s/bw7tbpo6A3qXYDz/download/2025-07-24-BeyondFossilFuels-Europe_Coal_Plants_Database.xlsx + fn: 20260731-BeyondFossilFuels-Europe_Coal_Plants_Database.xlsx + url: https://tubcloud.tu-berlin.de/s/qHfNEELYy6EnBSX/download/20260731-BeyondFossilFuels-Europe_Coal_Plants_Database.xlsx IRENA: net_capacity: true aggregated_units: true - fn: IRENASTAT_capacities_2000-2024.csv - # compiled from https://pxweb.irena.org/pxweb/en/IRENASTAT/IRENASTAT__Power%20Capacity%20and%20Generation/Country_ELECSTAT_2025_H2_PX.px/ - url: https://tubcloud.tu-berlin.de/s/dDS9erreKPNH4Ey/download/IRENASTAT_capacities_2000-2024.csv + fn: IRENASTAT_capacities_2000-2025.csv + # compiled from https://pxweb.irena.org/pxweb/en/IRENASTAT/IRENASTAT__Power%20Capacity%20and%20Generation/Country_ELECCAP_2026_H1_v-PX%201.px/ + url: https://tubcloud.tu-berlin.de/s/qALFkR8eB73FWA9/download/IRENASTAT_capacities_2000-2025.csv CARMA: net_capacity: false reliability_score: 1 @@ -87,15 +87,15 @@ CARMA: fn: Full_CARMA_2009_Dataset_1.csv ENTSOE: reliability_score: 5 - url: https://tubcloud.tu-berlin.de/s/N7qo3AGyRYZyisS/download/entsoe_transparency_platform_20250820.csv - fn: entsoe_transparency_platform_20250820.csv + url: https://tubcloud.tu-berlin.de/s/MfA7aHLKDZzNkeW/download/entsoe_transparency_platform_20260816-1122.csv + fn: entsoe_transparency_platform_20260816-1122.csv ENTSOE-EIC: url: https://eepublicdownloads.blob.core.windows.net/cio-lio/csv/W_eicCodes.csv fn: W_eicCodes.csv JRC: reliability_score: 5 fn: jrc-hydro-power-plant-database.csv - url: https://raw.githubusercontent.com/energy-modelling-toolkit/hydro-power-database/27e80f/data/jrc-hydro-power-plant-database.csv + url: https://raw.githubusercontent.com/energy-modelling-toolkit/hydro-power-database/3e8a837/data/jrc-hydro-power-plant-database.csv GEO: net_capacity: false reliability_score: 2 @@ -172,8 +172,8 @@ GGPT: net_capacity: false reliability_score: 6 status: ["operating", "retired", "construction"] - fn: Global-Oil-and-Gas-Plant-Tracker-GOGPT-August-2025.xlsx - url: https://tubcloud.tu-berlin.de/s/WrmNX5awNJFcXrQ/download/Global-Oil-and-Gas-Plant-Tracker-GOGPT-August-2025.xlsx + fn: Global-Oil-and-Gas-Plant-Tracker-GOGPT-January-2026.xlsx + url: https://tubcloud.tu-berlin.de/s/oTNEPzKcTZjdEZW/download/Global-Oil-and-Gas-Plant-Tracker-GOGPT-January-2026.xlsx GEM: # combined data set of all GEM trackers net_capacity: true @@ -182,15 +182,15 @@ GCPT: net_capacity: false reliability_score: 6 status: ["operating", "retired", "construction", "mothballed"] - fn: Global-Coal-Plant-Tracker-July-2025.xlsx - url: https://tubcloud.tu-berlin.de/s/ijzbscopNTgNB2r/download/Global-Coal-Plant-Tracker-July-2025.xlsx + fn: Global-Coal-Plant-Tracker-July-2026.xlsx + url: https://tubcloud.tu-berlin.de/s/xyZSXcd4QNSzDsy/download/Global-Coal-Plant-Tracker-July-2026.xlsx GGTPT: net_capacity: false reliability_score: 6 aggregated_units: false status: ["operating", "retired", "construction", "mothballed"] - fn: Geothermal-Power-Tracker-March-2025-Final.xlsx - url: https://tubcloud.tu-berlin.de/s/ypr3eL2K5kckAK4/download/Geothermal-Power-Tracker-March-2025-Final.xlsx + fn: Geothermal-Power-Tracker-March-2026-Final.xlsx + url: https://tubcloud.tu-berlin.de/s/spGtyE3oPZrMeB3/download/Geothermal-Power-Tracker-March-2026-Final.xlsx GWPT: net_capacity: false reliability_score: 6 @@ -213,27 +213,28 @@ GNPT: net_capacity: false reliability_score: 6 status: ["operating", "retired", "mothballed", "construction"] - fn: Global-Nuclear-Power-Tracker-July-2024.xlsx - url: https://tubcloud.tu-berlin.de/s/gXFim9EciRHrjeQ/download/Global-Nuclear-Power-Tracker-July-2024.xlsx + fn: Global-Nuclear-Power-Tracker-September-2025.xlsx + url: https://tubcloud.tu-berlin.de/s/NBX6G2eyZ3T97gq/download/Global-Nuclear-Power-Tracker-September-2025.xlsx GHPT: net_capacity: false reliability_score: 6 status: ["operating", "retired", "construction"] - fn: Global-Hydropower-Tracker-April-2025.xlsx - url: https://tubcloud.tu-berlin.de/s/aDyd3MJWZNgeEH4/download/Global-Hydropower-Tracker-April-2025.xlsx + fn: Global-Hydropower-Tracker-March-2026.xlsx + url: https://tubcloud.tu-berlin.de/s/feKWmAfzPtHPHN2/download/Global-Hydropower-Tracker-March-2026.xlsx MASTR: net_capacity: true reliability_score: 7 status: ["In Betrieb", "In Planung", "Endgültig stillgelegt", "Vorübergehend stillgelegt"] - capacity_threshold: 0.1 # all values below will be filtered out, given in MW - fn: bnetza_open_mastr_2025-02-09.zip - url: https://zenodo.org/records/14783581/files/bnetza_open_mastr_2025-02-09.zip + capacity_threshold: 0.05 # all values below will be filtered out, given in MW + fn: dataversion-2026-08-16.zip + url: https://tubcloud.tu-berlin.de/s/KJWoc8rwezoRkcQ/download/dataversion-2026-08-16.zip EESI: net_capacity: true reliability_score: 5 status: ["Operational"] # since no start years given - fn: european-energy-storage-inventory-20250817-2245.json - url: https://tubcloud.tu-berlin.de/s/5KqMDMZfb2pN3Aw/download/european-energy-storage-inventory-20250817-2245.json + fn: european-energy-storage-inventory-20260816-1026.json + # from https://ses.jrc.ec.europa.eu/storage-inventory-tool/api/projects + url: https://tubcloud.tu-berlin.de/s/ijiN2EWjHfg4NXf/download/european-energy-storage-inventory-20260816-1026.json GND: net_capacity: true reliability_score: 5 diff --git a/powerplantmatching/utils.py b/powerplantmatching/utils.py index 4b9396b..5ae37ee 100644 --- a/powerplantmatching/utils.py +++ b/powerplantmatching/utils.py @@ -9,6 +9,7 @@ import multiprocessing import os import re +import shutil from ast import literal_eval as liteval from importlib.metadata import version @@ -80,13 +81,17 @@ def get_raw_file(name, update=False, config=None, skip_retrieve=False): if (not os.path.exists(path) or update) and not skip_retrieve: url = df_config["url"] - logger.info(f"Retrieving data from {url}") - base_version = parse(version(__package__)).base_version - user_agent = f"{__package__}/{base_version}" - r = requests.get(url, headers={"User-Agent": user_agent}, timeout=60) - r.raise_for_status() - with open(path, "wb") as outfile: - outfile.write(r.content) + if os.path.exists(url): + logger.info(f"Copying data from local file {url}") + shutil.copyfile(url, path) + else: + logger.info(f"Retrieving data from {url}") + base_version = parse(version(__package__)).base_version + user_agent = f"{__package__}/{base_version}" + r = requests.get(url, headers={"User-Agent": user_agent}, timeout=60) + r.raise_for_status() + with open(path, "wb") as outfile: + outfile.write(r.content) return path diff --git a/pyproject.toml b/pyproject.toml index 63aeb04..4a4ef2f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ dependencies = [ "geopy", "xlrd", "unidecode", - "entsoe-py >=0.3.1", + "entsoe-py>=0.8", "deprecation", "tqdm", "openpyxl",