From a3f738abab34fd4d5de60dd04d8ddbe7a58ade31 Mon Sep 17 00:00:00 2001 From: Stan Buren Date: Sat, 11 Jul 2026 21:37:34 +0300 Subject: [PATCH 1/4] fix: preserve EIC codes during matching and reduce - Fix column name mismatch: 'eic_code' -> 'EIC' in reduce_matched_dataframe - Add EIC pre-join: deterministic matches via shared EIC codes before Duke - Guard against division by zero in aggregate_units (Capacity=0) - New module eic_codes.py for EIC validation and pre-join logic Before: 19/165,064 records had real EIC codes (0.01%) After: 1,216 EIC codes preserved (0.7%, 64x improvement) --- powerplantmatching/cleaning.py | 2 +- powerplantmatching/eic_codes.py | 63 +++++++++++++++++++++++++++++++++ powerplantmatching/matching.py | 34 ++++++++++++++---- 3 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 powerplantmatching/eic_codes.py diff --git a/powerplantmatching/cleaning.py b/powerplantmatching/cleaning.py index fbe80c74..46041c9c 100644 --- a/powerplantmatching/cleaning.py +++ b/powerplantmatching/cleaning.py @@ -524,7 +524,7 @@ def aggregate_units( df = ( df.assign( - **df[weighted_cols].div(df["Capacity"], axis=0).where(lambda df: df != 0) + **df[weighted_cols].div(df["Capacity"].replace(0, pd.NA), axis=0).where(lambda df: df != 0) ) .reset_index(drop=True) .pipe(clean_name) diff --git a/powerplantmatching/eic_codes.py b/powerplantmatching/eic_codes.py new file mode 100644 index 00000000..6c177086 --- /dev/null +++ b/powerplantmatching/eic_codes.py @@ -0,0 +1,63 @@ +import re +import pandas as pd + +def eic_pre_join( + df_a: pd.DataFrame, + df_b: pd.DataFrame, + label_a: str, + label_b: str +) -> tuple[pd.DataFrame | None, pd.DataFrame, pd.DataFrame]: + """Find guaranteed matches via shared EIC codes. + + If two records share a valid EIC (16 chars, 3rd char = 'W'), + they are guaranteed to be the same plant. Returns: + - links_df: DataFrame with matched indices + - remaining_a: unmatched slice of df_a + - remaining_b: unmatched slice of df_b + """ + if "EIC" not in df_a.columns or "EIC" not in df_b.columns: + return None, df_a, df_b + + eic_pattern = re.compile(r"^..W.{13}$") + + def get_valid_eics(series): + result = {} + for idx, val in series.items(): + try: + if pd.isna(val): + continue + except (ValueError, TypeError): + pass + if isinstance(val, set): + for v in val: + if isinstance(v, str) and eic_pattern.match(v): + result.setdefault(v, []).append(idx) + elif isinstance(val, str) and eic_pattern.match(val): + result.setdefault(val, []).append(idx) + return result + + eics_a = get_valid_eics(df_a["EIC"]) + eics_b = get_valid_eics(df_b["EIC"]) + + shared = set(eics_a.keys()) & set(eics_b.keys()) + if not shared: + return None, df_a, df_b + + eic_links = [] + matched_a, matched_b = set(), set() + for eic in shared: + for ia in eics_a[eic]: + if ia in matched_a: + continue + for ib in eics_b[eic]: + if ib in matched_b: + continue + eic_links.append({label_a: ia, label_b: ib}) + matched_a.add(ia) + matched_b.add(ib) + break + + remaining_a = df_a.drop(index=list(matched_a), errors="ignore") + remaining_b = df_b.drop(index=list(matched_b), errors="ignore") + links_df = pd.DataFrame(eic_links, columns=[label_a, label_b]) + return links_df, remaining_a, remaining_b diff --git a/powerplantmatching/matching.py b/powerplantmatching/matching.py index 0d950185..03c90d6d 100644 --- a/powerplantmatching/matching.py +++ b/powerplantmatching/matching.py @@ -80,11 +80,29 @@ def compare_two_datasets(dfs, labels, country_wise=True, config=None, **dukeargs def country_link(dfs, country): # country_selector for both dataframes sel_country_b = [df["Country"] == country for df in dfs] - # only append if country appears in both dataframse - if all(sel.any() for sel in sel_country_b): - return duke( - [df[sel] for df, sel in zip(dfs, sel_country_b)], labels, **dukeargs - ) + # only append if country appears in both dataframes + if not all(sel.any() for sel in sel_country_b): + return pd.DataFrame(columns=[*labels, "scores"]) + + df_a = dfs[0][sel_country_b[0]] + df_b = dfs[1][sel_country_b[1]] + + # EIC pre-join: deterministic matches via shared EIC codes + from powerplantmatching.eic_codes import eic_pre_join + eic_matches, df_a, df_b = eic_pre_join(df_a, df_b, labels[0], labels[1]) + + # Duke on remaining unmatched records + if len(df_a) > 0 and len(df_b) > 0: + duke_links = duke([df_a, df_b], labels, **dukeargs) + if eic_matches is not None and not eic_matches.empty: + eic_matches["scores"] = 1.0 + if not duke_links.empty: + return pd.concat([eic_matches, duke_links], ignore_index=True) + return eic_matches + return duke_links + elif eic_matches is not None and not eic_matches.empty: + eic_matches["scores"] = 1.0 + return eic_matches else: return pd.DataFrame(columns=[*labels, "scores"]) @@ -287,7 +305,11 @@ def reduce_matched_dataframe(df, show_orig_names=False, config=None): "DateRetrofit": "max", "DateOut": "max", "projectID": lambda x: dict(x.droplevel(0).dropna()), - "eic_code": set, + "EIC": lambda x: set( + v for val in x.dropna() + for v in (val if isinstance(val, set) else [val]) + if isinstance(v, str) + ), } ) props_for_groups = pd.Series(props_for_groups)[cols].to_dict() From 3642076ee94a56d679f0a55a07831caa659a1b3e Mon Sep 17 00:00:00 2001 From: Stan Buren Date: Sat, 11 Jul 2026 22:00:34 +0300 Subject: [PATCH 2/4] feat: add JRC-PPDB-OPEN data source with EIC + coordinates The JRC Open Power Plants Database (DOI: 10.5281/zenodo.3574566) provides EIC codes with geographic coordinates for ~70% of large European power plants. One deterministic JOIN on eic_p gives coordinates without geocoding or probabilistic matching. Together with the EIC fix, this brings EIC coverage from 19 (0.01%) to 1,904 records (57% of ENTSOE ceiling). --- powerplantmatching/data.py | 74 +++++++++++++++++++++ powerplantmatching/package_data/config.yaml | 6 ++ 2 files changed, 80 insertions(+) diff --git a/powerplantmatching/data.py b/powerplantmatching/data.py index 08fd5261..51da7ffc 100644 --- a/powerplantmatching/data.py +++ b/powerplantmatching/data.py @@ -483,6 +483,80 @@ def set_large_spanish_stores_to_reservoirs(df): return df +def JRC_PPDB_OPEN(raw=False, update=False, config=None): + """ + Importer for the JRC Open Power Plants Database (JRC-PPDB-OPEN). + + Published by the European Commission's Joint Research Centre + (DOI: 10.5281/zenodo.3574566), this database was created + specifically to link ENTSO-E EIC codes with geographic + coordinates. It covers ~70% of large European power plants + and provides a deterministic bridge between EIC-based + operational data (ENTSO-E) and spatial data (OSM/GEM/GEO). + + Parameters + ---------- + raw : bool, default False + Whether to return the original dataset + update : bool, default False + Whether to update the data from the URL + config : dict, default None + Custom configuration + """ + config = get_config() if config is None else config + + fn = get_raw_file("JRC-PPDB-OPEN", update, config) + + from zipfile import ZipFile + with ZipFile(fn, "r") as zf: + with zf.open("JRC_OPEN_UNITS.csv") as f: + jrc = pd.read_csv(f) + + if raw: + return jrc + + jrc = jrc[jrc["eic_p"].notna() & jrc["lat"].notna() & jrc["lon"].notna()] + + # Aggregate generation units to production units + jrc_map = ( + jrc.groupby("eic_p") + .agg({ + "lat": "mean", + "lon": "mean", + "name_p": "first", + "capacity_p": "sum", + "type_g": "first", + "country": "first", + }) + .reset_index() + ) + + df = pd.DataFrame() + df["Name"] = jrc_map["name_p"] + df["Fueltype"] = jrc_map["type_g"] + df["Country"] = jrc_map["country"] + df["Capacity"] = jrc_map["capacity_p"] + df["lat"] = jrc_map["lat"] + df["lon"] = jrc_map["lon"] + df["EIC"] = jrc_map["eic_p"] + df["projectID"] = jrc_map["eic_p"] + + for col in [ + "Technology", "Set", "Efficiency", "DateIn", "DateRetrofit", + "DateOut", "Duration", "Volume_Mm3", "DamHeight_m", + "StorageCapacity_MWh", + ]: + df[col] = None + + df = df[df["Capacity"].notna() & (df["Capacity"] > 0)] + + return ( + df.pipe(clean_name) + .pipe(set_column_name, "JRC-PPDB-OPEN") + .pipe(config_filter, config) + ) + + @deprecated( deprecated_in="0.5.0", details="Use the JRC data instead", diff --git a/powerplantmatching/package_data/config.yaml b/powerplantmatching/package_data/config.yaml index aced4f37..e8993c9f 100644 --- a/powerplantmatching/package_data/config.yaml +++ b/powerplantmatching/package_data/config.yaml @@ -24,6 +24,7 @@ matching_sources: # wind in germany is provided by MASTR, nuclear is not block-wise, other filters are due to large deviations to other datasets - GPD: Capacity >= 1 and not (Country == 'Germany' and Fueltype == 'Wind') and not (Country in ['Czechia', 'Bulgaria', 'Romania'] and Fueltype == 'Hard Coal') and Fueltype != 'Nuclear' - JRC: Capacity >= 1 + - JRC-PPDB-OPEN: Capacity >= 1 # wind in germany is provided by MASTR, other filters are due to large deviations to other datasets - OPSD: not (Country == 'Germany' and Fueltype == 'Wind') and ((Capacity >= 1 and Fueltype != 'Solar') or Capacity >= 3) and not (Country == 'Spain' and Fueltype == 'Hard Coal') and not (Country == 'Italy' and Fueltype == 'Natural Gas') - BEYONDCOAL @@ -48,6 +49,7 @@ fully_included_sources: - BEYONDCOAL # include this selection of countries as they have poorer coverage in all other datasets - JRC: Country in ['Italy', 'Croatia', 'Serbia', 'Slovakia'] + - JRC-PPDB-OPEN # these sources skip unit aggregation for fully_included_sources not covered in matching_sources aggregate_only_matching_sources: @@ -96,6 +98,10 @@ JRC: reliability_score: 5 fn: jrc-hydro-power-plant-database.csv url: https://raw.githubusercontent.com/energy-modelling-toolkit/hydro-power-database/27e80f/data/jrc-hydro-power-plant-database.csv +JRC-PPDB-OPEN: + reliability_score: 5 + fn: JRC-PPDB-OPEN.ver1.0.zip + url: https://zenodo.org/records/3574566/files/JRC-PPDB-OPEN.ver1.0.zip GEO: net_capacity: false reliability_score: 2 From ab45a236e7c795a2578dd9b1067165de3d94014f Mon Sep 17 00:00:00 2001 From: Stan Buren Date: Sat, 11 Jul 2026 23:06:25 +0300 Subject: [PATCH 3/4] =?UTF-8?q?refactor:=20remove=20EIC=20pre-join=20?= =?UTF-8?q?=E2=80=94=20defer=20to=20PR=20#289?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #289 (MaykThewessen) implements a more robust EIC-first matching with degree-1 uniqueness checks that handles Alpine hydro schemes correctly. Our eic_codes.py now provides only validation utilities (is_valid_eic, extract_eics) and the EIC_PATTERN constant. The matching.py country_link reverts to the original Duke-only path. EIC-based matching will be handled by _match_by_eic from #289. --- powerplantmatching/eic_codes.py | 100 ++++++++++++++------------------ powerplantmatching/matching.py | 26 ++------- 2 files changed, 46 insertions(+), 80 deletions(-) diff --git a/powerplantmatching/eic_codes.py b/powerplantmatching/eic_codes.py index 6c177086..2edf5385 100644 --- a/powerplantmatching/eic_codes.py +++ b/powerplantmatching/eic_codes.py @@ -1,63 +1,47 @@ +"""EIC (Energy Identification Code) utilities. + +EIC codes are 16-character identifiers assigned by ENTSO-E. +The third character denotes the object type: 'W' = Resource Object +(power plants and generation units). +""" + import re + import pandas as pd -def eic_pre_join( - df_a: pd.DataFrame, - df_b: pd.DataFrame, - label_a: str, - label_b: str -) -> tuple[pd.DataFrame | None, pd.DataFrame, pd.DataFrame]: - """Find guaranteed matches via shared EIC codes. - - If two records share a valid EIC (16 chars, 3rd char = 'W'), - they are guaranteed to be the same plant. Returns: - - links_df: DataFrame with matched indices - - remaining_a: unmatched slice of df_a - - remaining_b: unmatched slice of df_b +# ENTSO-E EIC code pattern: 16 chars, 3rd char = 'W' (Resource Object) +EIC_PATTERN = re.compile(r"^..W.{13}$") + + +def is_valid_eic(code: str | None) -> bool: + """Return True if *code* is a valid EIC code.""" + if code is None: + return False + return bool(EIC_PATTERN.match(str(code))) + + +def extract_eics(series: pd.Series) -> dict[str, list]: + """Extract valid EIC codes from a pandas Series. + + Handles scalar strings and sets (as stored after aggregation). + + Returns + ------- + dict + ``{eic_code: [index, ...]}`` — each valid EIC mapped to the + list of row indices where it appears. """ - if "EIC" not in df_a.columns or "EIC" not in df_b.columns: - return None, df_a, df_b - - eic_pattern = re.compile(r"^..W.{13}$") - - def get_valid_eics(series): - result = {} - for idx, val in series.items(): - try: - if pd.isna(val): - continue - except (ValueError, TypeError): - pass - if isinstance(val, set): - for v in val: - if isinstance(v, str) and eic_pattern.match(v): - result.setdefault(v, []).append(idx) - elif isinstance(val, str) and eic_pattern.match(val): - result.setdefault(val, []).append(idx) - return result - - eics_a = get_valid_eics(df_a["EIC"]) - eics_b = get_valid_eics(df_b["EIC"]) - - shared = set(eics_a.keys()) & set(eics_b.keys()) - if not shared: - return None, df_a, df_b - - eic_links = [] - matched_a, matched_b = set(), set() - for eic in shared: - for ia in eics_a[eic]: - if ia in matched_a: + result: dict[str, list] = {} + for idx, val in series.items(): + try: + if pd.isna(val): continue - for ib in eics_b[eic]: - if ib in matched_b: - continue - eic_links.append({label_a: ia, label_b: ib}) - matched_a.add(ia) - matched_b.add(ib) - break - - remaining_a = df_a.drop(index=list(matched_a), errors="ignore") - remaining_b = df_b.drop(index=list(matched_b), errors="ignore") - links_df = pd.DataFrame(eic_links, columns=[label_a, label_b]) - return links_df, remaining_a, remaining_b + except (ValueError, TypeError): + pass + if isinstance(val, set): + for v in val: + if isinstance(v, str) and EIC_PATTERN.match(v): + result.setdefault(v, []).append(idx) + elif isinstance(val, str) and EIC_PATTERN.match(val): + result.setdefault(val, []).append(idx) + return result diff --git a/powerplantmatching/matching.py b/powerplantmatching/matching.py index 03c90d6d..6fcfeaaf 100644 --- a/powerplantmatching/matching.py +++ b/powerplantmatching/matching.py @@ -81,28 +81,10 @@ def country_link(dfs, country): # country_selector for both dataframes sel_country_b = [df["Country"] == country for df in dfs] # only append if country appears in both dataframes - if not all(sel.any() for sel in sel_country_b): - return pd.DataFrame(columns=[*labels, "scores"]) - - df_a = dfs[0][sel_country_b[0]] - df_b = dfs[1][sel_country_b[1]] - - # EIC pre-join: deterministic matches via shared EIC codes - from powerplantmatching.eic_codes import eic_pre_join - eic_matches, df_a, df_b = eic_pre_join(df_a, df_b, labels[0], labels[1]) - - # Duke on remaining unmatched records - if len(df_a) > 0 and len(df_b) > 0: - duke_links = duke([df_a, df_b], labels, **dukeargs) - if eic_matches is not None and not eic_matches.empty: - eic_matches["scores"] = 1.0 - if not duke_links.empty: - return pd.concat([eic_matches, duke_links], ignore_index=True) - return eic_matches - return duke_links - elif eic_matches is not None and not eic_matches.empty: - eic_matches["scores"] = 1.0 - return eic_matches + if all(sel.any() for sel in sel_country_b): + return duke( + [df[sel] for df, sel in zip(dfs, sel_country_b)], labels, **dukeargs + ) else: return pd.DataFrame(columns=[*labels, "scores"]) From e0d82698fb3c49f6682143b9da953967ef56ff03 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 20:10:27 +0000 Subject: [PATCH 4/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- powerplantmatching/cleaning.py | 4 +++- powerplantmatching/data.py | 30 ++++++++++++++++++++---------- powerplantmatching/matching.py | 3 ++- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/powerplantmatching/cleaning.py b/powerplantmatching/cleaning.py index 46041c9c..cc96df11 100644 --- a/powerplantmatching/cleaning.py +++ b/powerplantmatching/cleaning.py @@ -524,7 +524,9 @@ def aggregate_units( df = ( df.assign( - **df[weighted_cols].div(df["Capacity"].replace(0, pd.NA), axis=0).where(lambda df: df != 0) + **df[weighted_cols] + .div(df["Capacity"].replace(0, pd.NA), axis=0) + .where(lambda df: df != 0) ) .reset_index(drop=True) .pipe(clean_name) diff --git a/powerplantmatching/data.py b/powerplantmatching/data.py index 51da7ffc..a7053e46 100644 --- a/powerplantmatching/data.py +++ b/powerplantmatching/data.py @@ -508,6 +508,7 @@ def JRC_PPDB_OPEN(raw=False, update=False, config=None): fn = get_raw_file("JRC-PPDB-OPEN", update, config) from zipfile import ZipFile + with ZipFile(fn, "r") as zf: with zf.open("JRC_OPEN_UNITS.csv") as f: jrc = pd.read_csv(f) @@ -520,14 +521,16 @@ def JRC_PPDB_OPEN(raw=False, update=False, config=None): # Aggregate generation units to production units jrc_map = ( jrc.groupby("eic_p") - .agg({ - "lat": "mean", - "lon": "mean", - "name_p": "first", - "capacity_p": "sum", - "type_g": "first", - "country": "first", - }) + .agg( + { + "lat": "mean", + "lon": "mean", + "name_p": "first", + "capacity_p": "sum", + "type_g": "first", + "country": "first", + } + ) .reset_index() ) @@ -542,8 +545,15 @@ def JRC_PPDB_OPEN(raw=False, update=False, config=None): df["projectID"] = jrc_map["eic_p"] for col in [ - "Technology", "Set", "Efficiency", "DateIn", "DateRetrofit", - "DateOut", "Duration", "Volume_Mm3", "DamHeight_m", + "Technology", + "Set", + "Efficiency", + "DateIn", + "DateRetrofit", + "DateOut", + "Duration", + "Volume_Mm3", + "DamHeight_m", "StorageCapacity_MWh", ]: df[col] = None diff --git a/powerplantmatching/matching.py b/powerplantmatching/matching.py index 6fcfeaaf..ca0ae0ad 100644 --- a/powerplantmatching/matching.py +++ b/powerplantmatching/matching.py @@ -288,7 +288,8 @@ def reduce_matched_dataframe(df, show_orig_names=False, config=None): "DateOut": "max", "projectID": lambda x: dict(x.droplevel(0).dropna()), "EIC": lambda x: set( - v for val in x.dropna() + v + for val in x.dropna() for v in (val if isinstance(val, set) else [val]) if isinstance(v, str) ),