diff --git a/config/cgm_worker/config_areas_mapping.json b/config/cgm_worker/config_areas_mapping.json new file mode 100644 index 00000000..c8d18d80 --- /dev/null +++ b/config/cgm_worker/config_areas_mapping.json @@ -0,0 +1,389 @@ +[ + { + "area.eic": "10Y1001A1001A39I", + "area.code": "EE", + "area.name": "Estonia", + "party.name": "ELERING", + "party.eic": "10X1001A1001A39W", + "opdm.shortname": "ee", + "opdm.longname": "EE" + }, + { + "area.eic": "10Y1001A1001A59C", + "area.code": "IE_NI", + "area.name": "Ireland/UK (Northern Ireland)", + "party.name": "EIRGRID", + "party.eic": "10X1001A1001A531", + "opdm.shortname": null, + "opdm.longname": null + }, + { + "area.eic": "10Y1001A1001A990", + "area.code": "MD", + "area.name": "Moldova", + "party.name": "TRANSELECTRICA", + "party.eic": "10XRO-TEL------2", + "opdm.shortname": null, + "opdm.longname": null + }, + { + "area.eic": "10Y1001C--000182", + "area.code": "UA", + "area.name": "Ukraine", + "party.name": "UA_WPS", + "party.eic": "10X1001C--00001X", + "opdm.shortname": null, + "opdm.longname": null + }, + { + "area.eic": "10Y1001C--00100H", + "area.code": "XK", + "area.name": "Kosovo", + "party.name": "KOSOVA-TSMO", + "party.eic": "10XRKS-KOSTT-007", + "opdm.shortname": "kostt", + "opdm.longname": "KOSTT" + }, + { + "area.eic": "10YAL-KESH-----5", + "area.code": "AL", + "area.name": "Albania", + "party.name": "OST", + "party.eic": "10XAL-KESH-----J", + "opdm.shortname": "al", + "opdm.longname": "AL" + }, + { + "area.eic": "10YAT-APG------L", + "area.code": "AT", + "area.name": "Austria", + "party.name": "APG", + "party.eic": "10XAT-APG------Z", + "opdm.shortname": "apg", + "opdm.longname": "APG" + }, + { + "area.eic": "10YBA-JPCC-----D", + "area.code": "BA", + "area.name": "Bosnia and Herzegovina", + "party.name": "NOSBIH", + "party.eic": "10XBA-JPCCZEKC-K", + "opdm.shortname": "ba", + "opdm.longname": "BA" + }, + { + "area.eic": "10YBE----------2", + "area.code": "BE", + "area.name": "Belgium", + "party.name": "ELIA", + "party.eic": "10X1001A1001A094", + "opdm.shortname": "be", + "opdm.longname": "BE" + }, + { + "area.eic": "10YCA-BULGARIA-R", + "area.code": "BG", + "area.name": "Bulgaria", + "party.name": "ESO", + "party.eic": "10XBG-ESO------A", + "opdm.shortname": "bg", + "opdm.longname": "BG" + }, + { + "area.eic": "10YCH-SWISSGRIDZ", + "area.code": "CH", + "area.name": "Switzerland", + "party.name": "SWISSGRID", + "party.eic": "10XCH-SWISSGRIDC", + "opdm.shortname": "ch", + "opdm.longname": "CH" + }, + { + "area.eic": "10YCS-CG-TSO---S", + "area.code": "ME", + "area.name": "Montenegro", + "party.name": "CGES", + "party.eic": "10XCS-CG-TSO---5", + "opdm.shortname": "cges", + "opdm.longname": "CGES" + }, + { + "area.eic": "10YCS-SERBIATSOV", + "area.code": "RS", + "area.name": "Serbia", + "party.name": "EMS", + "party.eic": "10XCS-SERBIATSO8", + "opdm.shortname": "ems", + "opdm.longname": "EMS" + }, + { + "area.eic": "10YCZ-CEPS-----N", + "area.code": "CZ", + "area.name": "Czech Republic", + "party.name": "CEPS", + "party.eic": "10XCZ-CEPS-GRIDE", + "opdm.shortname": "ceps", + "opdm.longname": "CEPS" + }, + { + "area.eic": "10YDE-ENBW-----N", + "area.code": "DE-TRANSNETBW", + "area.name": "DE_Transnet", + "party.name": "D4", + "party.eic": "10XDE-ENBW--TNGX", + "opdm.shortname": "d4", + "opdm.longname": "D4" + }, + { + "area.eic": "10YDE-EON------1", + "area.code": "DE-TENNET_DE", + "area.name": "DE_Tennet", + "party.name": "TTG", + "party.eic": "10XDE-EON-NETZ-C", + "opdm.shortname": "d2", + "opdm.longname": "D2" + }, + { + "area.eic": "10YDE-RWENET---I", + "area.code": "DE-AMPRION-SCHED", + "area.name": "DE_Amprion", + "party.name": "D7", + "party.eic": "10XDE-RWENET---W", + "opdm.shortname": "amprion", + "opdm.longname": "Amprion" + }, + { + "area.eic": "10YDE-VE-------2", + "area.code": "DE-50HERTZ", + "area.name": "DE_50Hertz", + "party.name": "50Hertz", + "party.eic": "10XDE-VE-TRANSMK", + "opdm.shortname": "50hertz", + "opdm.longname": "50Hertz" + }, + { + "area.eic": "10YDK-1--------W", + "area.code": "DK1", + "area.name": "Denmark DK1", + "party.name": "DKW", + "party.eic": "10X1001A1001A248", + "opdm.shortname": "d1", + "opdm.longname": "D1" + }, + { + "area.eic": "10YDK-2--------M", + "area.code": "DK2", + "area.name": "Denmark DK2", + "party.name": "DKE", + "party.eic": "10X1001A1001A248", + "opdm.shortname": "dk", + "opdm.longname": "DK" + }, + { + "area.eic": "10YES-REE------0", + "area.code": "ES", + "area.name": "Spain", + "party.name": "REE", + "party.eic": "10XES-REE------E", + "opdm.shortname": "es", + "opdm.longname": "ES" + }, + { + "area.eic": "10YFI-1--------U", + "area.code": "FI", + "area.name": "Finland", + "party.name": "FI", + "party.eic": "10X1001A1001A264", + "opdm.shortname": "fi", + "opdm.longname": "FI" + }, + { + "area.eic": "10YFR-RTE------C", + "area.code": "FR", + "area.name": "France", + "party.name": "RTEFRANCE", + "party.eic": "10XFR-RTE------Q", + "opdm.shortname": "fr", + "opdm.longname": "FR" + }, + { + "area.eic": "10YGB----------A", + "area.code": "GB", + "area.name": "United Kingdom", + "party.name": "NGET", + "party.eic": "10X1001A1001A515", + "opdm.shortname": "gb", + "opdm.longname": "GB" + }, + { + "area.eic": "10YGR-HTSO-----Y", + "area.code": "GR", + "area.name": "Greece", + "party.name": "IPTO", + "party.eic": "10XGR-HTSO-----B", + "opdm.shortname": "gr", + "opdm.longname": "GR" + }, + { + "area.eic": "10YHR-HEP------M", + "area.code": "HR", + "area.name": "Croatia", + "party.name": "HOPS", + "party.eic": "10XHR-HEP-OPS--A", + "opdm.shortname": "hops", + "opdm.longname": "HOPS" + }, + { + "area.eic": "10YHU-MAVIR----U", + "area.code": "HU", + "area.name": "Hungary", + "party.name": "MAVIR", + "party.eic": "10X1001A1001A329", + "opdm.shortname": "hu", + "opdm.longname": "HU" + }, + { + "area.eic": "10YIT-GRTN-----B", + "area.code": "IT", + "area.name": "Italy", + "party.name": "TERNA", + "party.eic": "10X1001A1001A345", + "opdm.shortname": "it", + "opdm.longname": "IT" + }, + { + "area.eic": "10YLT-1001A0008Q", + "area.code": "LT", + "area.name": "Lithuania", + "party.name": "LITGRID", + "party.eic": "10X1001A1001A55Y", + "opdm.shortname": "litgrid", + "opdm.longname": "Litgrid" + }, + { + "area.eic": "10YLU-CEGEDEL-NQ", + "area.code": "LU", + "area.name": "Luxembourg", + "party.name": "CREOS", + "party.eic": "21X000000001333E", + "opdm.shortname": "creos", + "opdm.longname": "Creos" + }, + { + "area.eic": "10YLV-1001A00074", + "area.code": "LV", + "area.name": "Latvia", + "party.name": "AST", + "party.eic": "10X1001A1001B54W", + "opdm.shortname": "ast", + "opdm.longname": "AST" + }, + { + "area.eic": "10YMA-ONE------O", + "area.code": "MO", + "area.name": "Morocco", + "party.name": "REE", + "party.eic": "10XES-REE------E", + "opdm.shortname": null, + "opdm.longname": null + }, + { + "area.eic": "10YMK-MEPSO----8", + "area.code": "MK", + "area.name": "Macedonia", + "party.name": "MEPSO", + "party.eic": "10XMK-MEPSO----M", + "opdm.shortname": "mepso", + "opdm.longname": "MEPSO" + }, + { + "area.eic": "10YNL----------L", + "area.code": "NL", + "area.name": "Netherlands", + "party.name": "TTN", + "party.eic": "10X1001A1001A361", + "opdm.shortname": "nl", + "opdm.longname": "NL" + }, + { + "area.eic": "10YNO-0--------C", + "area.code": "NO", + "area.name": "Norway", + "party.name": "STATNETT", + "party.eic": "10X1001A1001A38Y", + "opdm.shortname": "no", + "opdm.longname": "NO" + }, + { + "area.eic": "10YPL-AREA-----S", + "area.code": "PL", + "area.name": "Poland", + "party.name": "PSE", + "party.eic": "10XPL-TSO------P", + "opdm.shortname": "pl", + "opdm.longname": "PL" + }, + { + "area.eic": "10YPT-REN------W", + "area.code": "PT", + "area.name": "Portugal", + "party.name": "REN", + "party.eic": "10XPT-REN------9", + "opdm.shortname": "pt", + "opdm.longname": "PT" + }, + { + "area.eic": "10YRO-TEL------P", + "area.code": "RO", + "area.name": "Romania", + "party.name": "Transelectrica", + "party.eic": "10XRO-TEL------2", + "opdm.shortname": "ro", + "opdm.longname": "RO" + }, + { + "area.eic": "10YSE-1--------K", + "area.code": "SE", + "area.name": "Sweden", + "party.name": "SVK", + "party.eic": "10X1001A1001A418", + "opdm.shortname": "se", + "opdm.longname": "SE" + }, + { + "area.eic": "10YSI-ELES-----O", + "area.code": "SI", + "area.name": "Slovenia", + "party.name": "ELES", + "party.eic": "10XSI-ELES-----1", + "opdm.shortname": "eles", + "opdm.longname": "ELES" + }, + { + "area.eic": "10YSK-SEPS-----K", + "area.code": "SK", + "area.name": "Slovak Republic", + "party.name": "SEPS", + "party.eic": "10XSK-SEPS-GRIDB", + "opdm.shortname": "seps", + "opdm.longname": "SEPS" + }, + { + "area.eic": "10YTR-TEIAS----W", + "area.code": "TR", + "area.name": "Turkey", + "party.name": "TEIAS", + "party.eic": "10XTR-TEIAS----9", + "opdm.shortname": "teias", + "opdm.longname": "TEIAS" + }, + { + "area.eic": "10YUA-WEPS-----0", + "area.code": "UA", + "area.name": "Ukraine", + "party.name": "UKRENERGO", + "party.eic": "10X1001C--00001X", + "opdm.shortname": "ua", + "opdm.longname": "UA" + } +] \ No newline at end of file diff --git a/emf/common/integrations/object_storage/models.py b/emf/common/integrations/object_storage/models.py index 2d63b4c1..35ef50ee 100644 --- a/emf/common/integrations/object_storage/models.py +++ b/emf/common/integrations/object_storage/models.py @@ -1,269 +1,283 @@ -from emf.common.integrations import object_storage -from aniso8601 import parse_datetime -import logging -import pandas -import sys -from emf.common.integrations import opdm - -logger = logging.getLogger(__name__) - - -def compile_query(metadata: dict, filter: str | None): - - match_and_term_list = [] - for key, value in metadata.items(): - if isinstance(value, list): - match_and_term_list.append({"terms": {key: value}}) - else: - match_and_term_list.append({"match": {key: value}}) - - if filter: - query = {"bool": {"must": match_and_term_list, "filter": {"range": {"pmd:scenarioDate": {"gte": filter}}}}} - else: - query = {"bool": {"must": match_and_term_list}} - - return query - - -def fetch_unique_values(metadata_query: dict, - field: str, - query_filter: str | None = None, - index: str = object_storage.ELASTIC_MODELS_INDEX, - page_size: int = 10000, - ): - - # Validate index definition to be able to search all index by pattern - if "*" not in index: - index = f"{index}*" - - agg_name = "values_page" - unique_vals = [] - after_key = None - - # Compile query - query = compile_query(metadata=metadata_query, filter=query_filter) - - while True: - comp = { - "size": page_size, - "sources": [ - {"val": {"terms": {"field": field}}} - ] - } - if after_key: - comp["after"] = after_key - - body = { - "size": 0, - "query": query, - "aggs": { - agg_name: { - "composite": comp - } - } - } - - response = object_storage.elastic_service.client.search(index=index, body=body) - buckets = response["aggregations"][agg_name]["buckets"] - unique_vals.extend(b["key"]["val"] for b in buckets) - - # Page on if there's more - if "after_key" in response["aggregations"][agg_name]: - after_key = response["aggregations"][agg_name]["after_key"] - else: - break - - return unique_vals - - -def query_data(metadata_query: dict, - query_filter: str | None = None, - index: str = object_storage.ELASTIC_MODELS_INDEX, - return_payload: bool = False, - size: str = '10000', - sort: dict | None = None, - scroll: str = '1m', - ): - """ - Queries Elasticsearch based on provided metadata queries. - - Args: - metadata_query (dict): A dictionary containing metadata fields and their values to be queried. - query_filter (dict): Optional. A dictionary specifying parameters by which to filter the query. - index (str): The index to query data from. Defaults to ELASTIC_MODELS_INDEX from config variables. - return_payload (bool): Optional. If True, retrieves the full content for each hit. - Defaults to False. - - Returns: - list: A list of dictionaries containing the retrieved content from Elasticsearch. - - Note: - The function constructs an Elasticsearch query based on the provided metadata_query. - It retrieves data from the specified index and processes the response to extract content. - - Example: - To query data with metadata fields 'TSO' and 'timeHorizon' and return payload: - metadata_query = {"pmd:TSO": "TERNA", "pmd:timeHorizon": "2D"} - response = query_data(metadata_query, return_payload=True) - """ - - # Create elastic query syntax - # { - # "bool": { - # "must": [ - # {"match": {"pmd:TSO": "TERNA"}}, - # {"terms": {"pmd:timeHorizon": ["01", "02"]}} - # ] - # } - # } - - # Validate index definition to be able to search all index by pattern - if "*" not in index: - index = f"{index}*" - - # Compile query - query = compile_query(metadata=metadata_query, filter=query_filter) - - # Return query results - response = object_storage.elastic_service.client.search(index=index, query=query, size=size, sort=sort, scroll=scroll) - scroll_id = response['_scroll_id'] - hits = response["hits"]["hits"] - content_list = [content["_source"] for content in hits] # TODO - consider optimize without this loop - while len(hits) > 0: - response = object_storage.elastic_service.client.scroll(scroll_id=scroll_id, scroll=scroll) - hits = response["hits"]["hits"] - if hits: - content_list.extend([content["_source"] for content in hits]) - - if return_payload: - for num, item in enumerate(content_list): - content_list[num] = get_content(item) - - # Delete scroll after retrieving data - object_storage.elastic_service.client.clear_scroll(scroll_id=scroll_id) - - return content_list - - -def get_content(metadata: dict): - """ - Retrieves content data from MinIO based on metadata information. - - Args: - metadata (dict): A dictionary containing metadata information. - - Returns: - list: A list of dictionaries representing content components with updated 'DATA' field. - - Note: - It expects metadata to contain 'opde:Component' information. - For each component, it downloads data from MinIO and updates the 'DATA' field in the component dictionary. - """ - - logger.info(f"Getting content of metadata object from MinIO: {metadata['opde:Id']}") - bucket_name = metadata.get("minio-bucket", "opdm-data") # by default use "opdm-data" bucket if missing in meta - logger.debug(f"S3 storage bucket used: {bucket_name}") - components_received = [] - for component in metadata["opde:Component"]: - content_reference = component.get("opdm:Profile").get("pmd:content-reference") - logger.info(f"Downloading object: {bucket_name}/{content_reference}") - content = object_storage.minio_service.download_object(bucket_name, content_reference) - component["opdm:Profile"]["DATA"] = content - components_received.append(bool(content)) # collect boolean flags of received components - - if not all(components_received): # at least one is False - logger.warning(f"[FALLBACK] At least some content did not exist in MinIO storage, requesting from OPDM...") - metadata = opdm.OPDM().download_object(metadata) # TODO maybe to make OPDM connection instance globally - - return metadata - - -def get_latest_boundary(): - - logger.info(f"Retrieving latest boundary set") - - # Query data from ELK - boundaries = query_data({"opde:Object-Type.keyword": "BDS"}) - - # Convert to dataframe for sorting out the latest boundary - boundary_data = pandas.DataFrame(boundaries) - - # Convert date and version to respective formats - boundary_data['date_time'] = pandas.to_datetime(boundary_data['pmd:scenarioDate'], format='ISO8601') - boundary_data['version'] = pandas.to_numeric(boundary_data['pmd:versionNumber']) - - # Sort out official boundary - official_boundary_data = boundary_data[boundary_data["opde:Context"] == {'opde:IsOfficial': 'true'}] - - # Get the latest boundary meta - latest_boundary_meta = boundaries[list(official_boundary_data.sort_values(["date_time", "version"], ascending=False).index)[0]] - - # Download the latest boundary - return get_content(metadata=latest_boundary_meta) - - -def get_latest_models_and_download(time_horizon: str, - scenario_date: str, - valid: bool = True, - tso: str | None = None, - object_type: str = 'IGM', - data_source: str | None = None - ): - - logger.info(f"Retrieving latest network models of type: {object_type} [source: {data_source}]") - - meta = {'pmd:validFrom': f"{parse_datetime(scenario_date):%Y%m%dT%H%MZ}", - 'pmd:timeHorizon': time_horizon, - 'opde:Object-Type': object_type, - "data-source": data_source} - - if tso: - meta['pmd:TSO'] = tso - - if valid: - meta["valid"] = valid - - if data_source: - meta["data-source"] = data_source - - if time_horizon.upper() == "ID": - meta['pmd:timeHorizon'] = [f"{i:02d}" for i in range(1, 32)] - # TODO - This is not a nice solution, needs to be moved to somewhere more close to business function as this can change - - models_metadata_raw = query_data(metadata_query=meta, return_payload=False) - - models_downloaded = [] - - if models_metadata_raw: - # Sort for highest timeHorizon (for intraday) and for highest version - models = pandas.DataFrame(models_metadata_raw) - latest_models = models.sort_values(["pmd:timeHorizon", "pmd:versionNumber"], ascending=[True, False]).groupby("pmd:modelPartReference").first() - - for model in latest_models.to_dict("records"): - try: - models_downloaded.append(get_content(metadata=model)) - except: - logger.error(f"Could not download model for {time_horizon} {scenario_date} {model['pmd:TSO']}") - logger.error(sys.exc_info()) - else: - logger.warning(f"Models not available on Object Storage") - - return models_downloaded - - -if __name__ == "__main__": - logging.basicConfig(stream=sys.stdout, - format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', - level=logging.INFO) - - test_query = {"pmd:TSO": "TERNA", - "pmd:timeHorizon": "2D", - "pmd:scenarioDate": "2025-02-15T22:30:00Z", - } - test_filter = "now-2w" - test_response = query_data(test_query, query_filter=test_filter, return_payload=True) - - #models = get_latest_models_and_download("1D", '20240526T1530Z', valid=False) - models = get_latest_models_and_download("ID", '20240522T1530Z', valid=True) - bds = get_latest_boundary() +from emf.common.integrations import object_storage +from aniso8601 import parse_datetime +import logging +import pandas +import sys +from emf.common.integrations import opdm + +logger = logging.getLogger(__name__) + + +def compile_query(metadata: dict, filter: str | None): + + """Creates the query in the correct syntax acceptable by Elastic""" + + match_and_term_list = [] + for key, value in metadata.items(): + if isinstance(value, list): + match_and_term_list.append({"terms": {key: value}}) + else: + match_and_term_list.append({"match": {key: value}}) + # Looks like the filter was made to handle value ranges not to match by TSO + # (and the filter is expected to be the scenario date range specifically) + # So the initial desired TSO filter was added to the metadata dict, not the filter + if filter: + query = {"bool": {"must": match_and_term_list, "filter": {"range": {"pmd:scenarioDate": {"gte": filter}}}}} + else: + query = {"bool": {"must": match_and_term_list}} + + return query + + +def fetch_unique_values(metadata_query: dict, + field: str, + query_filter: str | None = None, + index: str = object_storage.ELASTIC_MODELS_INDEX, + page_size: int = 10000, + ): + + # Validate index definition to be able to search all index by pattern + if "*" not in index: + index = f"{index}*" + + agg_name = "values_page" + unique_vals = [] + after_key = None + + # Compile query + query = compile_query(metadata=metadata_query, filter=query_filter) + + while True: + comp = { + "size": page_size, + "sources": [ + {"val": {"terms": {"field": field}}} + ] + } + if after_key: + comp["after"] = after_key + + body = { + "size": 0, + "query": query, + "aggs": { + agg_name: { + "composite": comp + } + } + } + + response = object_storage.elastic_service.client.search(index=index, body=body) + buckets = response["aggregations"][agg_name]["buckets"] + unique_vals.extend(b["key"]["val"] for b in buckets) + + # Page on if there's more + if "after_key" in response["aggregations"][agg_name]: + after_key = response["aggregations"][agg_name]["after_key"] + else: + break + + return unique_vals + + +def query_data(metadata_query: dict, + query_filter: str | None = None, + index: str = object_storage.ELASTIC_MODELS_INDEX, + return_payload: bool = False, + size: str = '10000', + sort: dict | None = None, + scroll: str = '1m', + ): + """ + Queries Elasticsearch based on provided metadata queries. + + Args: + metadata_query (dict): A dictionary containing metadata fields and their values to be queried. + query_filter (dict): Optional. A dictionary specifying parameters by which to filter the query. + index (str): The index to query data from. Defaults to ELASTIC_MODELS_INDEX from config variables. + return_payload (bool): Optional. If True, retrieves the full content for each hit. + Defaults to False. + + Returns: + list: A list of dictionaries containing the retrieved content from Elasticsearch. + + Note: + The function constructs an Elasticsearch query based on the provided metadata_query. + It retrieves data from the specified index and processes the response to extract content. + + Example: + To query data with metadata fields 'TSO' and 'timeHorizon' and return payload: + metadata_query = {"pmd:TSO": "TERNA", "pmd:timeHorizon": "2D"} + response = query_data(metadata_query, return_payload=True) + """ + + # Create elastic query syntax + # { + # "bool": { + # "must": [ + # {"match": {"pmd:TSO": "TERNA"}}, + # {"terms": {"pmd:timeHorizon": ["01", "02"]}} + # ] + # } + # } + + # Validate index definition to be able to search all index by pattern + if "*" not in index: + index = f"{index}*" + + # Compile query + query = compile_query(metadata=metadata_query, filter=query_filter) + + # Return query results + response = object_storage.elastic_service.client.search(index=index, query=query, size=size, sort=sort, scroll=scroll) + scroll_id = response['_scroll_id'] + hits = response["hits"]["hits"] + content_list = [content["_source"] for content in hits] # TODO - consider optimize without this loop + while len(hits) > 0: + response = object_storage.elastic_service.client.scroll(scroll_id=scroll_id, scroll=scroll) + hits = response["hits"]["hits"] + if hits: + content_list.extend([content["_source"] for content in hits]) + + if return_payload: + for num, item in enumerate(content_list): + content_list[num] = get_content(item) + + # Delete scroll after retrieving data + object_storage.elastic_service.client.clear_scroll(scroll_id=scroll_id) + + return content_list + + +def get_content(metadata: dict): + """ + Retrieves content data from MinIO based on metadata information. + + Args: + metadata (dict): A dictionary containing metadata information. + + Returns: + list: A list of dictionaries representing content components with updated 'DATA' field. + + Note: + It expects metadata to contain 'opde:Component' information. + For each component, it downloads data from MinIO and updates the 'DATA' field in the component dictionary. + """ + + logger.info(f"Getting content of metadata object from MinIO: {metadata['opde:Id']}") + bucket_name = metadata.get("minio-bucket", "opdm-data") # by default use "opdm-data" bucket if missing in meta + logger.debug(f"S3 storage bucket used: {bucket_name}") + components_received = [] + for component in metadata["opde:Component"]: + content_reference = component.get("opdm:Profile").get("pmd:content-reference") + logger.info(f"Downloading object: {bucket_name}/{content_reference}") + content = object_storage.minio_service.download_object(bucket_name, content_reference) + component["opdm:Profile"]["DATA"] = content + components_received.append(bool(content)) # collect boolean flags of received components + + if not all(components_received): # at least one is False + logger.warning(f"[FALLBACK] At least some content did not exist in MinIO storage, requesting from OPDM...") + metadata = opdm.OPDM().download_object(metadata) # TODO maybe to make OPDM connection instance globally + + return metadata + + +def get_latest_boundary(): + + logger.info(f"Retrieving latest boundary set") + + # Query data from ELK + boundaries = query_data({"opde:Object-Type.keyword": "BDS"}) + + # Convert to dataframe for sorting out the latest boundary + boundary_data = pandas.DataFrame(boundaries) + + # Convert date and version to respective formats + boundary_data['date_time'] = pandas.to_datetime(boundary_data['pmd:scenarioDate'], format='ISO8601') + boundary_data['version'] = pandas.to_numeric(boundary_data['pmd:versionNumber']) + + # Sort out official boundary + official_boundary_data = boundary_data[boundary_data["opde:Context"] == {'opde:IsOfficial': 'true'}] + + # Get the latest boundary meta + latest_boundary_meta = boundaries[list(official_boundary_data.sort_values(["date_time", "version"], ascending=False).index)[0]] + + # Download the latest boundary + return get_content(metadata=latest_boundary_meta) + + +def get_latest_models_and_download(time_horizon: str, + scenario_date: str, + valid: bool = True, + tso: list[str] | None = None, # this used to be str but I don't see how that would work so changed to list + object_type: str = 'IGM', + data_source: str | None = None + ): + """Gathers the conditions for filtering for the models metadata query, performs the query and returns the metadatas""" + logger.info(f"Retrieving latest network models of type: {object_type} [source: {data_source}]") + + meta = {'pmd:validFrom': f"{parse_datetime(scenario_date):%Y%m%dT%H%MZ}", + 'pmd:timeHorizon': time_horizon, + 'opde:Object-Type': object_type, + "data-source": data_source} + + if tso: + meta['pmd:TSO.keyword'] = tso # but note that this is now a list not a str like the others + + if valid: + meta["valid"] = valid + + if data_source: + meta["data-source"] = data_source + + if time_horizon.upper() == "ID": + meta['pmd:timeHorizon'] = [f"{i:02d}" for i in range(1, 32)] + # TODO - This is not a nice solution, needs to be moved to somewhere more close to business function as this can change + + models_metadata_raw = query_data(metadata_query=meta, return_payload=False) + + models_downloaded = [] + + if models_metadata_raw: + # Sort for highest timeHorizon (for intraday) and for highest version + models = pandas.DataFrame(models_metadata_raw) + latest_models = models.sort_values(["pmd:timeHorizon", "pmd:versionNumber"], ascending=[True, False]).groupby("pmd:modelPartReference").first() + + for model in latest_models.to_dict("records"): + try: + models_downloaded.append(get_content(metadata=model)) + except: + logger.error(f"Could not download model for {time_horizon} {scenario_date} {model['pmd:TSO']}") + logger.error(sys.exc_info()) + else: + logger.warning(f"Models not available on Object Storage") + + return models_downloaded + + +if __name__ == "__main__": + logging.basicConfig(stream=sys.stdout, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', + level=logging.INFO) + + #test_query = {"pmd:TSO": "TERNA", + # "pmd:timeHorizon": "2D", + # "pmd:scenarioDate": "2026-07-16T22:30:00Z", + # } + time_horizon="1D" + object_type="IGM" + scenario_date = "20260716T1530Z" + meta = {'pmd:validFrom': f"{parse_datetime(scenario_date):%Y%m%dT%H%MZ}", + 'pmd:timeHorizon': time_horizon, + 'opde:Object-Type': object_type, + "data-source": None, + "pmd:TSO": ["LITGRID", "AST", "PSE"]} + + test_query = compile_query(meta, filter=None) + test_filter = None #"now-2w" + test_response = query_data(test_query, query_filter=test_filter, return_payload=True) + + #models = get_latest_models_and_download("1D", '20240526T1530Z', valid=False) + #models = get_latest_models_and_download("ID", '20260716T1530Z', valid=True) + #bds = get_latest_boundary() logger.info("Test script finished") \ No newline at end of file diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index 0d0d09ac..b581a076 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -1,1105 +1,1125 @@ -import json -import math -import numpy as np -from xml.sax.expatreader import version -import pandas as pd -import pypowsybl -import logging -import sys -import datetime -import triplets -import uuid -import config -import xml.etree.ElementTree as ET -from emf.common.config_parser import parse_app_properties -from emf.common.integrations import elastic -from emf.model_merger import temporary -from emf.common.helpers.time import parse_datetime -from emf.common.helpers.loadflow import get_model_outages, get_network_elements -from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, filename_from_opdm_metadata -from emf.common.helpers.utils import sanitize_nan - - -logger = logging.getLogger(__name__) - -parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.post_processing) - - -def is_valid_uuid(uuid_value): - """ - Checks if input is uuid value - For merged SV profile the output uuid can be combination of several existing uuids - :param uuid_value: input value - :return - """ - try: - uuid.UUID(str(uuid_value)) - return True - except ValueError: - return False - - -def export_merged_model(network: pypowsybl.network, - opdm_object_meta: dict, - profiles: list[str] | None = None, - cgm_convention: bool = True, - ): - - # Define which profiles to export - if profiles: - profiles = ",".join(profiles) - else: - profiles = "SV,SSH,TP,EQ" - - # Define whether export using CGM official structure - cgm_export_flag = "False" - if cgm_convention: - cgm_export_flag = "True" - - # Define base name for exported files - file_base_name = filename_from_opdm_metadata(metadata=opdm_object_meta) - - # Define CGMES export parameters - parameters = { - "iidm.export.cgmes.modeling-authority-set": opdm_object_meta['pmd:modelingAuthoritySet'], - "iidm.export.cgmes.base-name": file_base_name, - "iidm.export.cgmes.profiles": profiles, - # For missing instances like "SupplyStation" - "iidm.export.cgmes.topology-kind": 'NODE_BREAKER', - # cgmes-fix-all-invalid-ids fixes non-standard uuid's. Can cause danglingReference errors - # "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids", # identity, cgmes, cgmes-fix-all-invalid-ids - "iidm.export.cgmes.export-sv-injections-for-slacks": "False", - # False sets all boundary flows to zero causing Kirchhoff 1st law and SvPowerFlowBranchInstances2 errors - # "iidm.export.cgmes.export-boundary-power-flows": "False", - "iidm.export.cgmes.cgm_export": cgm_export_flag, - } - - # Export to bytes object - bytes_object = network.save_to_binary_buffer(format="CGMES", parameters=parameters) - bytes_object.name = f"{file_base_name}_{uuid.uuid4()}.zip" - logger.info(f"Exported merged model to {bytes_object.name}") - - # TODO set correct naming of exported files - # a = triplets.rdf_parser.find_all_xml([bytes_object]) - - return bytes_object - - -def create_merged_model_opdm_object(object_id: str, - time_horizon: str, - merging_entity: str, - merging_area: str, - scenario_date: datetime.datetime, - mas: str, - version: str = "001", - profile: str = "SV", - content_type: str = "CGMES", - ): - opdm_object_meta = { - 'opde:Object-Type': 'CGM', - 'pmd:fullModel_ID': object_id, - 'pmd:creationDate': f"{datetime.datetime.now(datetime.UTC):%Y-%m-%dT%H:%M:%S.%fZ}", - 'pmd:timeHorizon': time_horizon, - 'pmd:cgmesProfile': profile, - 'pmd:contentType': content_type, - 'pmd:modelPartReference': "-".join([merging_entity, merging_area]), - 'pmd:mergingEntity': merging_entity, - 'pmd:Area': merging_area, - 'pmd:validFrom': f"{parse_datetime(scenario_date):%Y%m%dT%H%MZ}", - 'pmd:modelingAuthoritySet': mas, - "pmd:isFullModel": "true", - 'pmd:scenarioDate': f"{parse_datetime(scenario_date):%Y-%m-%dT%H:%M:00Z}", - 'pmd:modelid': object_id, - 'pmd:description': f""" - {time_horizon} - pypowsybl_{pypowsybl.__version__} - {merging_entity} - Model: Simplification of reality for given need. - """, - 'pmd:versionNumber': f"{int(version):03d}", - } - - return opdm_object_meta - - -def update_header_from_opdm_object(data: pd.DataFrame, opdm_object: dict): - return triplets.cgmes_tools.update_FullModel_from_dict(data, metadata={ - "Model.version": f"{int(opdm_object['pmd:versionNumber']):03d}", - "Model.created": f"{parse_datetime(opdm_object['pmd:creationDate']):%Y-%m-%dT%H:%M:%S.%fZ}", - "Model.mergingEntity": opdm_object['pmd:mergingEntity'], - "Model.domain": opdm_object['pmd:Area'], - "Model.scenarioTime": f"{parse_datetime(opdm_object['pmd:scenarioDate']):%Y-%m-%dT%H:%M:00Z}", - "Model.description": opdm_object['pmd:description'], - "Model.processType": opdm_object['pmd:timeHorizon'], - }) - - -def update_merged_model_sv(sv_data: bytes, opdm_object_meta: dict): - - # Load SV profile data - sv_data = pd.read_RDF([sv_data]) - - # Update rdfxml header from opdm object metadata - sv_data = update_header_from_opdm_object(data=sv_data, opdm_object=opdm_object_meta) - - # Update file name at 'label' key - sv_data.set_VALUE_at_KEY(key='label', value=filename_from_opdm_metadata(opdm_object_meta, file_type="xml")) - - sv_data = triplets.cgmes_tools.update_FullModel_from_filename(sv_data) - - # Check and fix SV id if necessary - updated_sv_id_map = {} - for old_id in sv_data.query("KEY == 'Type' and VALUE == 'FullModel'").ID.unique(): - if not is_valid_uuid(old_id): - new_id = str(uuid.uuid4()) - updated_sv_id_map[old_id] = new_id - logger.warning(f"SV profile id {old_id} is not valid, assigning: {new_id}") - sv_data = sv_data.replace(updated_sv_id_map) - - return sv_data - - -def load_ssh(input_data: pd.DataFrame | list): - """ - Loads in ssh profiles from list of profiles or takes the slice from dataframe - :param input_data: list of profiles or dataframe - :return dataframe of ssh data - """ - if not isinstance(input_data, pd.DataFrame): - ssh_data = load_opdm_objects_to_triplets(input_data, "SSH") - else: - ssh_files = (input_data[(input_data['KEY'] == 'label') & - (input_data['VALUE'].str.upper().str.contains('SSH'))][['INSTANCE_ID']] - .drop_duplicates()) - ssh_data = input_data.merge(ssh_files, on='INSTANCE_ID') - ssh_data = triplets.cgmes_tools.update_FullModel_from_filename(ssh_data) - return ssh_data - - -def create_updated_ssh(models_as_triplets: pd.DataFrame | list, - sv_data: pd.DataFrame, - opdm_object_meta: dict, - input_models: list = None, - ): - # TODO rewrite to use pypowsybl exported SSH - - ### SSH ## - - # Load original SSH data to created updated SSH - ssh_file_data = input_models or models_as_triplets - ssh_data = load_ssh(ssh_file_data) - - # Update SSH Model.scenarioTime - ssh_data.set_VALUE_at_KEY('Model.scenarioTime', opdm_object_meta['pmd:scenarioDate']) - - # Load full original data to fix issues - # data = load_opdm_data(original_models) - # terminals = data.type_tableview("Terminal") - - # Update SSH data from SV - ssh_update_map = [ - { - "from_class": "SvPowerFlow", - "from_ID": "Terminal.ConductingEquipment", - "from_attribute": "SvPowerFlow.p", - "to_attribute": "EnergyConsumer.p", - }, - { - "from_class": "SvPowerFlow", - "from_ID": "Terminal.ConductingEquipment", - "from_attribute": "SvPowerFlow.q", - "to_attribute": "EnergyConsumer.q", - }, - { - "from_class": "SvPowerFlow", - "from_ID": "Terminal.ConductingEquipment", - "from_attribute": "SvPowerFlow.p", - "to_attribute": "RotatingMachine.p", - }, - { - "from_class": "SvPowerFlow", - "from_ID": "Terminal.ConductingEquipment", - "from_attribute": "SvPowerFlow.q", - "to_attribute": "RotatingMachine.q", - }, - { - "from_class": "SvTapStep", - "from_ID": "SvTapStep.TapChanger", - "from_attribute": "SvTapStep.position", - "to_attribute": "TapChanger.step", - }, - { - "from_class": "SvShuntCompensatorSections", - "from_ID": "SvShuntCompensatorSections.ShuntCompensator", - "from_attribute": "SvShuntCompensatorSections.sections", - "to_attribute": "ShuntCompensator.sections", - }, - { - "from_class": "SvPowerFlow", - "from_ID": "Terminal.ConductingEquipment", - "from_attribute": "SvPowerFlow.p", - "to_attribute": "EquivalentInjection.p" - }, - { - "from_class": "SvPowerFlow", - "from_ID": "Terminal.ConductingEquipment", - "from_attribute": "SvPowerFlow.q", - "to_attribute": "EquivalentInjection.q" - } - ] - # Load terminal from original data - terminals = models_as_triplets.type_tableview("Terminal") - - # Update - for update in ssh_update_map: - # logger.info(f"Updating: {update['from_attribute']} -> {update['to_attribute']}") - source_data = sv_data.type_tableview(update['from_class']).reset_index(drop=True) - - # Merge with terminal, if needed - if terminal_reference := [column_name if ".Terminal" in column_name else None for column_name in source_data.columns][0]: - source_data = source_data.merge(terminals, left_on=terminal_reference, right_on='ID') - logger.debug(f"Added Terminals to {update['from_class']}") - - ssh_data = ssh_data.update_triplet_from_triplet(source_data.rename(columns={ - update['from_ID']: 'ID', - update['from_attribute']: update['to_attribute']} - )[['ID', update['to_attribute']]].set_index('ID').tableview_to_triplet(), add=False) - - # Generate new UUID for updated SSH - updated_ssh_id_map = {} - for OLD_ID in ssh_data.query("KEY == 'Type' and VALUE == 'FullModel'").ID.unique(): - NEW_ID = str(uuid.uuid4()) - updated_ssh_id_map[OLD_ID] = NEW_ID - logger.info(f"Assigned new UUID for updated SSH: {OLD_ID} -> {NEW_ID}") - - # Update SSH ID-s - ssh_data = ssh_data.replace(updated_ssh_id_map) - - # Update in SV SSH references - sv_data = sv_data.replace(updated_ssh_id_map) - - # Add SSH supersedes reference to old SSH - ssh_supersedes_data = pd.DataFrame([{"ID": item[1], "KEY": "Model.Supersedes", "VALUE": item[0]} for item in updated_ssh_id_map.items()]) - ssh_supersedes_data['INSTANCE_ID'] = ssh_data.query("KEY == 'Type'").merge(ssh_supersedes_data.ID)['INSTANCE_ID'] - ssh_data = ssh_data.update_triplet_from_triplet(ssh_supersedes_data) - - # Update SSH metadata - ssh_data = update_header_from_opdm_object(ssh_data, opdm_object_meta) - - # Update SSH filenames - filename_mask = "{scenarioTime:%Y%m%dT%H%MZ}_{processType}_{mergingEntity}-{domain}-{forEntity}_{messageType}_{version:03d}" - ssh_data = triplets.cgmes_tools.update_filename_from_FullModel(ssh_data, filename_mask=filename_mask) - - return sv_data, ssh_data, opdm_object_meta - - -def ensure_paired_equivalent_injection_compatibility(network: pypowsybl.network): - """Where there are paired boundary points, equivalent injections need to be modified to comply - LEVEL7 rule PairedEICompatibility - - Set P and Q to 0 - so that no additional consumption or production is on tie line - """ - logger.info("Configuring paired boundary points equivalent injections: p0/q0 = 0.0") - boundary_lines = network.get_boundary_lines(all_attributes=True) - paired_boundary_lines = boundary_lines[boundary_lines['paired'] == True] - if paired_boundary_lines.empty: - logger.warning(f"No paired boundary lines found in network model") - return network - - # Set p0/q0 to 0 for all paired boundary lines - _updated_p0 = pd.Series(0, index=paired_boundary_lines.index) - _updated_q0 = pd.Series(0, index=paired_boundary_lines.index) - network.update_boundary_lines(id=paired_boundary_lines.index, p0=_updated_p0, q0=_updated_q0) - - return network - - -def ensure_paired_boundary_line_connectivity(network: pypowsybl.network): - logger.info("Aligning paired boundary lines connection status") - boundary_lines = network.get_boundary_lines(all_attributes=True) - # Add cim:Tieflow attribute to boundary lines - boundary_lines['isTieflow'] = boundary_lines.index.isin(network.get_areas_boundaries()["element"]) - paired_boundary_lines = boundary_lines[boundary_lines['paired'] == True] - if paired_boundary_lines.empty: - logger.warning(f"No paired boundary lines found in network model") - return network - - # Identify boundary lines pairs where the 'connected' status is inconsistent within each pairing_key group - group = paired_boundary_lines.groupby('pairing_key') - mask_connected = group['connected'].transform(lambda s: s.nunique() > 1) - mask_tieflow = group['isTieflow'].transform( lambda s: s.nunique() > 1) - - mismatched_boundary_lines_con = paired_boundary_lines[mask_connected] - logger.info(f"Boundary lines with non-matching connection status: {mismatched_boundary_lines_con['pairing_key'].unique().tolist()}") - - mismatched_boundary_lines_tie = paired_boundary_lines[mask_tieflow] - logger.info(f"Boundary lines with non-matching cim:Tieflow: {mismatched_boundary_lines_tie['pairing_key'].unique().tolist()}") - - mismatched_boundary_lines = pd.concat([mismatched_boundary_lines_con, mismatched_boundary_lines_tie]) - - # Set all mismatched lines to disconnected (False) - _connected = pd.Series(data=False, index=mismatched_boundary_lines.index) - network.update_boundary_lines(id=mismatched_boundary_lines.index, connected=_connected) - - # Log each change - for i, row in mismatched_boundary_lines.iterrows(): - logger.info(f"Changed status of boundary line {row['name']}: {row['connected']} -> False") - - return network - - -def generate_merge_report(merged_model: object, task: dict): - """ - Creates JSON type report of pypowsybl loadflow results - - Args: - merged_model: merged pypowsybl network - task: task object dict - Returns: - dict: report of merge results - """ - report = merged_model.__dict__ - - # Pop out pypowsybl network - network = report.pop('network') - - # Include task data - report.update({'@timestamp': task.get('@timestamp'), - '@process_id': task.get('process_id'), - '@run_id': task.get('run_id'), - '@job_id': task.get('job_id'), - '@task_id': task.get('@id'), - '@time_horizon': task['task_properties'].get('time_horizon'), - '@scenario_timestamp': task['task_properties'].get('timestamp_utc'), - '@version': int(task['task_properties'].get('version')), - 'merge_type': task['task_properties'].get('merge_type'), - 'merge_entity': task['task_properties'].get('merging_entity'), - }) - - # Include buses count in each component - buses = get_network_elements(network, pypowsybl.network.ElementType.BUS) - buses_by_component = buses.connected_component.value_counts() - for component in report['loadflow']: - component['buses'] = buses_by_component.to_dict().get(component['connected_component_num']) - - # Count network components/islands - report['component_count'] = len(report['loadflow']) - - # Set trustability tag - report.update(evaluate_trustability(report, task['task_properties'])) - - return sanitize_nan(report) - - -def evaluate_trustability(report, properties) -> dict: - - reason = None - if properties["merge_type"] == "BA": - # Evaluate model trustability based on defined config and report keys - report_keys = ['scaled', 'replaced', 'outages'] - property_keys = ['scaling', 'replacement'] - - # Inline logic functions - key_true = lambda key: lambda d: bool(d.get(key)) - all_ = lambda *rules: lambda d: all(rule(d) for rule in rules) - all_none = lambda *keys, exclude=None: lambda d: all(d.get(k) is None for k in keys if k != exclude) - - # Compose conditions - config_all_true = all_(*(key_true(k) for k in property_keys)) - success_all_true = all_(*(key_true(k) for k in report_keys)) - success_all_none = all_none(*report_keys, exclude='scaled') # Scaling is never in None state - - # Evaluate logic - config_enabled = config_all_true(properties) - success_all_true = success_all_true(report) - success_all_none = success_all_none(report) - scaled_correctly = report['scaled'] - - reason_map = { - "scaled": "scaling failed", - "replaced": "replacement failed", - "outages": "outage fixing failed", - } - - # Decide trust level - if config_enabled and success_all_none and scaled_correctly: - trustability = "trusted" - elif config_enabled and success_all_true: - trustability = "semi-trusted" - else: - trustability = "untrusted" - if not config_enabled: - reason = "config is disabled" - else: - # From reason map get correct reason - for key, value in report.items(): - if key in reason_map and not report[key]: - reason = reason_map[key] - else: - trustability = 'not_evaluated' - - return {"trustability": trustability, "untrustability_reason": reason} - - -def filter_models(models: list, included_models: list | str = None, excluded_models: list | str = None, filter_on: str = 'pmd:TSO'): - """ - Filters the list of models to include or to exclude specific tsos if they are given. - If included is defined, excluded is not used - :param models: list of igm models - :param included_models: list or string of tso names, if given, only matching models are returned - :param excluded_models: list or string of tso names, if given, matching models will be discarded - :return updated list of igms - """ - - included_models = [included_models] if isinstance(included_models, str) else included_models - excluded_models = [excluded_models] if isinstance(excluded_models, str) else excluded_models - - if included_models: - logger.info(f"Models to be included: {included_models}") - elif excluded_models: - logger.info(f"Models to be excluded: {excluded_models}") - else: - logger.info(f"Including all available models: {[model['pmd:TSO'] for model in models]}") - return models - - filtered_models = [] - - for model in models: - - if included_models: - if model[filter_on] not in included_models: - logger.info(f"Excluded {model[filter_on]}") - continue - - elif excluded_models: - if model[filter_on] in excluded_models: - logger.info(f"Excluded {model[filter_on]}") - continue - - logger.info(f"Included {model[filter_on]}") - filtered_models.append(model) - - return filtered_models - - -def filter_models_by_acnp(models: list, merged_model, acnp_dict, acnp_threshold, conform_load_factor): - - def is_within_acnp_deadband(model): - tso = model.get('pmd:TSO') - if not tso or tso not in acnp_dict: - logger.error(f"TSO '{tso}' not found in acnp dict, skipping filtering") - return True - acnp = acnp_dict[tso] - return abs(float(model['ac_net_position']) - float(acnp)) <= float(acnp_threshold) - - def is_within_conformload_deadband(model): - tso = model.get('pmd:TSO') - if not tso or tso not in acnp_dict: - logger.error(f"TSO '{tso}' not found in acnp dict, skipping filtering") - return True - acnp = acnp_dict[tso] - expected_load = model['sum_conform_load'] * float(conform_load_factor) - return expected_load > abs(float(model['ac_net_position']) - float(acnp)) - - excluded_tso_ids = set() - - # ACNP deadband filter - filtered_models = [model for model in models if is_within_acnp_deadband(model)] - excluded_tsos= [ - {'tso': model['pmd:TSO'], 'reason': 'acnp-outside-schedule-deadband'} - for model in models - if model['pmd:TSO'] not in [fm['pmd:TSO'] for fm in filtered_models] - and model['pmd:TSO'] not in excluded_tso_ids - ] - if excluded_tsos: - excluded_tso_ids.update(model['tso'] for model in excluded_tsos) - logger.warning(f"Exluded TSO due to incorrect schedules: {excluded_tso_ids}") - merged_model.excluded.extend(excluded_tsos) - - # Conformload filter - final_models = [model for model in filtered_models if is_within_conformload_deadband(model)] - excluded_tsos= [ - {'tso': model['pmd:TSO'], 'reason': 'conform-load-outside-schedule-difference'} - for model in filtered_models - if model['pmd:TSO'] not in [fm['pmd:TSO'] for fm in final_models] - and model['pmd:TSO'] not in excluded_tso_ids - ] - if excluded_tsos: - excluded_tso_ids.update(model['tso'] for model in excluded_tsos) - logger.warning(f"Exluded TSO due to incorrect conform load: {excluded_tso_ids}") - merged_model.excluded.extend(excluded_tsos) - - return final_models - - -def filter_replacements_by_acnp(models: pd.DataFrame, acnp_dict, acnp_threshold, conform_load_factor): - """ - Drop replacement models whose AC net position deviates too much from the - scheduled ACNP for their TSO, or whose conform load cannot cover that - deviation. - - Inputs are sanitized so malformed data (missing columns, non-numeric or - missing values, an invalid acnp_dict/threshold/factor) never raises - - rows that cannot be evaluated are kept unfiltered instead of failing. - """ - required_columns = {'pmd:TSO', 'ac_net_position', 'sum_conform_load'} - if models.empty or not isinstance(acnp_dict, dict) or not required_columns.issubset(models.columns): - return models - try: - threshold, load_factor = float(acnp_threshold), float(conform_load_factor) - except (TypeError, ValueError): - return models - - acnp = pd.to_numeric(models['pmd:TSO'].map(acnp_dict), errors='coerce') - deviation = (pd.to_numeric(models['ac_net_position'], errors='coerce') - acnp).abs() - load = pd.to_numeric(models['sum_conform_load'], errors='coerce') - - keep = acnp.isna() | (deviation.notna() & (deviation <= threshold) & (load * load_factor > deviation)) - return models[keep] - - -def update_model_outages(merged_model: object, tso_list: list, scenario_datetime: str, time_horizon: str): - - area_map = {"LITGRID": "Lithuania", "AST": "Latvia", "ELERING": "Estonia"} - outage_areas = [area_map.get(item, item) for item in tso_list] - - elk_service = elastic.Elastic() - - # Get outage eic-mrid mapping - mrid_map = elk_service.get_docs_by_query(index='config-network*', query={"match_all": {}}, size=10000) - mrid_map['mrid'] = mrid_map['mrid'].str.lstrip('_') - - # Get latest UAP parse date - if time_horizon == 'MO': - merge_type = "Month" - elif time_horizon == 'YR': - merge_type = "Year" - else: - merge_type = "Week" - - body = {"size": 1, "query": {"bool": {"must": [{"match": {"Merge": merge_type}}]}}, - "sort": [{"reportParsedDate": {"order": "desc"}}], "fields": ["reportParsedDate"]} - last_uap_version = elk_service.client.search(index='opc-outages-baltics*', body=body)['hits']['hits'][0]['fields']['reportParsedDate'][0] - - # Query for latest outage UAP - uap_query = {"bool": {"must": [{"match": {"reportParsedDate": f"{last_uap_version}"}}, - {"match": {"Merge": merge_type}}]}} - uap_outages = elk_service.get_docs_by_query(index='opc-outages-baltics*', query=uap_query, size=10000) - - # Filter out incorrect elements - uap_outages['mrid'] = uap_outages['mrid'].replace("None", pd.NA) - uap_outages = uap_outages[uap_outages["asset_type"] != "PROD"] - - # Map missing mrid by eic - lookup = mrid_map.set_index(mrid_map[['eic', 'mrid']].columns[0])[mrid_map[['eic', 'mrid']].columns[1]] - uap_outages.loc[:, 'mrid'] = uap_outages['mrid'].fillna(uap_outages['eic'].map(lookup)) - - unmapped_outages = uap_outages[uap_outages['mrid'].isna()] - # Exception rule for old LitPol element - unmapped_outages = unmapped_outages[unmapped_outages['eic'] != "10T-LT-PL-000037"] - if not unmapped_outages.empty: - logger.warning(f"Unable to map following outage mRIDs: {unmapped_outages['name'].values}") - - # Filter outages according to model scenario date and replaced area - filtered_outages = uap_outages[(uap_outages['start_date'] <= scenario_datetime) & (uap_outages['end_date'] >= scenario_datetime)] - filtered_outages = filtered_outages[filtered_outages['Area'].isin(outage_areas)] - mapped_outages = filtered_outages[~filtered_outages['mrid'].isna()] - - # Get disconnected elements in network model - model_outages = pd.DataFrame(get_model_outages(network=merged_model.network)) - mapped_model_outages = pd.merge(model_outages, mrid_map, left_on='grid_id', right_on='mrid', how='inner') - model_area_map = {"LITGRID": "LT", "AST": "LV", "ELERING": "EE"} - model_outage_areas = [model_area_map.get(item, item) for item in tso_list] - filtered_model_outages = mapped_model_outages[mapped_model_outages['country'].isin(model_outage_areas)] - - # Include cross-border lines for reconnection (both boundary lines) - boundary_lines = get_network_elements(network=merged_model.network, - element_type=pypowsybl.network.ElementType.BOUNDARY_LINE).reset_index(names=['grid_id']) - border_lines = boundary_lines[boundary_lines['pairing_key'].isin(model_outages['pairing_key'])] - relevant_border_lines = border_lines[border_lines['country'].isin(model_outage_areas)] - # Removing any BRELL lines - relevant_border_lines = relevant_border_lines[~relevant_border_lines['lineEnergyIdentificationCodeEIC'].str.contains('RU')] - additional_boundary_lines = boundary_lines[boundary_lines['pairing_key'].isin(relevant_border_lines['pairing_key'])] - - # Merged dataframe of network elements to be reconnected - filtered_model_outages = pd.concat([filtered_model_outages, additional_boundary_lines]).drop_duplicates(subset='grid_id') - filtered_model_outages = filtered_model_outages.where(pd.notnull(filtered_model_outages), None) - - # rename columns - filtered_model_outages = filtered_model_outages.copy()[['name', 'grid_id', 'eic']].rename(columns={'grid_id': 'mrid'}) - mapped_outages = mapped_outages[['name', 'mrid', 'eic']].copy() - mapped_outages.loc[:, 'mrid'] = mapped_outages['mrid'].str.lstrip('_') - - logger.info(f"Updating outages in merged model areas: {model_outage_areas}") - - # Reconnecting outages from network-config list - outages_updated = {} - filtered_model_outages["eic"] = (filtered_model_outages["eic"].astype(object).where(filtered_model_outages["eic"].notna(), None)) - for index, outage in filtered_model_outages.iterrows(): - try: - if merged_model.network.connect(outage['mrid']): - logger.info(f"Successfully reconnected: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages = True - outage_dict = outage.to_dict() - outage_dict.update({'status': 'connected'}) - outages_updated[outage_dict['mrid']] = outage_dict - else: - if uap_outages['grid_id'].str.contains(outage['mrid']).any(): - logger.info(f"Element is already connected: {outage['name']} [mrid: {outage['mrid']}]") - else: - logger.error(f"Failed to connect element: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - except Exception as e: - logger.error((e, outage['name'])) - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - merged_model.outages = False - continue - - # Applying outages from UAP - mapped_outages["eic"] = (mapped_outages["eic"].astype(object).where(mapped_outages["eic"].notna(), None)) - for index, outage in mapped_outages.iterrows(): - try: - if merged_model.network.disconnect(outage['mrid']): - logger.info(f"Successfully disconnected: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages = True - outage_dict = outage.to_dict() - outage_dict.update({'status': 'disconnected'}) - outages_updated[outage_dict['mrid']] = outage_dict - else: - if uap_outages['mrid'].str.contains(("_" + outage['mrid'])).any(): - logger.info(f"Element is already in outage: {outage['name']} [mrid: {outage['mrid']}]") - else: - logger.error(f"Failed to disconnect element: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - except Exception as e: - logger.error((e, outage['name'], outage['mrid'])) - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - merged_model.outages = False - continue - - # Keep only important keys of updated outages - merged_model.outages_updated = list(outages_updated.values()) - - if merged_model.outages_unmapped: - merged_model.outages = False - - # Sanitise NaN values in merge report - merged_model.outages_updated = [{k: None if isinstance(v, float) and math.isnan(v) else v for k, v in d.items()} - for d in merged_model.outages_updated] - merged_model.outages_unmapped = [{k: None if isinstance(v, float) and math.isnan(v) else v for k, v in d.items()} - for d in merged_model.outages_unmapped] - - return merged_model - - -def set_intraday_time_horizon(scenario_datetime, task_creation_time): - """ - Finds time difference between task creation time and scenario timestamp. Converts it to hours and finds the hour - number corresponding to intraday run (number of hours that scenario timestamp is ahead from task creation time) - Here are multiple ways to calculate (must keep in mind start and end times/dates) - 1) Ceil: f"{math.ceil((_scenario_datetime - _task_creation_time).seconds / 3600):02d}" - 2) Round: f"{int((_scenario_datetime - _task_creation_time).seconds / 3600):02d}" - 3) Floor: f"{math.floor((_scenario_datetime - _task_creation_time).seconds / 3600):02d}" - Take into account date change - 4) Min(max(Ceil)): max(math.ceil((time_diff.days * 24 * 3600 + time_diff.seconds) / 3600), 1) - 5) Min(max(Round)): max(int((time_diff.days * 24 * 3600 + time_diff.seconds) / 3600), 1) - 6) Min(max(Floor)): max(math.floor((time_diff.days * 24 * 3600 + time_diff.seconds) / 3600), 1) - :param scenario_datetime: scenario timestamp for intraday run - :param task_creation_time: timestamp when the task was created - :return: time horizon for intraday run as a string - """ - max_time_horizon_value = 36 - calculated_time_horizon = '01' # DEFAULT VALUE, CHANGE THIS - _task_creation_time = parse_datetime(task_creation_time, keep_timezone=False) - _scenario_datetime = parse_datetime(scenario_datetime, keep_timezone=False) - time_diff = _scenario_datetime - _task_creation_time - if 0 <= time_diff.days <= 1: - time_horizon_actual = math.floor((time_diff.days * 24 * 3600 + time_diff.seconds) / 3600) - # just in case cut it to bigger than 1 once again - time_horizon_actual = max(time_horizon_actual, 1) - if time_horizon_actual <= max_time_horizon_value: - calculated_time_horizon = f"{time_horizon_actual:02d}" - return calculated_time_horizon - - -def check_net_interchanges(cgm_sv_data, cgm_ssh_data, original_models): - """ - An attempt to calculate the net interchange 2 values and check them against those provided in ssh profiles - :param cgm_sv_data: merged sv profile - :param cgm_ssh_data: merged ssh profile - :param original_models: igms in triplets - :param fix_errors: injects new calculated flows into merged ssh profiles - :param threshold: specify threshold if needed - :return (updated) ssh profiles - """ - try: - control_areas = (original_models.type_tableview('ControlArea') - .rename_axis('ControlArea') - .reset_index())[['ControlArea', 'ControlArea.netInterchange', 'ControlArea.pTolerance', - 'IdentifiedObject.energyIdentCodeEic', 'IdentifiedObject.name']] - except KeyError: - control_areas = original_models.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() - ssh_areas = cgm_ssh_data.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() - control_areas = control_areas.merge(ssh_areas, on='ControlArea')[['ControlArea', 'ControlArea.netInterchange', - 'ControlArea.pTolerance', - 'IdentifiedObject.energyIdentCodeEic', - 'IdentifiedObject.name']] - tie_flows = (original_models.type_tableview('TieFlow') - .rename_axis('TieFlow').rename(columns={'TieFlow.ControlArea': 'ControlArea', - 'TieFlow.Terminal': 'Terminal'}) - .reset_index())[['ControlArea', 'Terminal', 'TieFlow.positiveFlowIn']] - tie_flows = tie_flows.merge(control_areas[['ControlArea']], on='ControlArea') - try: - terminals = (original_models.type_tableview('Terminal') - .rename_axis('Terminal').reset_index())[['Terminal', 'ACDCTerminal.connected']] - except KeyError: - terminals = (original_models.type_tableview('Terminal') - .rename_axis('Terminal').reset_index())[['Terminal']] - tie_flows = tie_flows.merge(terminals, on='Terminal') - try: - power_flows_pre = (original_models.type_tableview('SvPowerFlow') - .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) - .reset_index())[['Terminal', 'SvPowerFlow.p']] - tie_flows = tie_flows.merge(power_flows_pre, on='Terminal', how='left') - except Exception as error: - logger.error(f"Was not able to get tie flows from original models with exception: {error}") - power_flows_post = (cgm_sv_data.type_tableview('SvPowerFlow') - .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) - .reset_index())[['Terminal', 'SvPowerFlow.p']] - - tie_flows = tie_flows.merge(power_flows_post, on='Terminal', how='left', - suffixes=('_pre', '_post')) - try: - tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p_pre', 'SvPowerFlow.p_post']] - .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) - .rename_axis('ControlArea').reset_index()) - except KeyError: - tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p']] - .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) - .rename_axis('ControlArea').reset_index()) - tie_flows_grouped = tie_flows_grouped.rename(columns={'SvPowerFlow.p': 'SvPowerFlow.p_post'}) - tie_flows_grouped = control_areas.merge(tie_flows_grouped, on='ControlArea') - - net_interchange_errors = tie_flows_grouped.loc[ - tie_flows_grouped['ControlArea.netInterchange'].ne(tie_flows_grouped['SvPowerFlow.p_post'])] - - if not net_interchange_errors.empty: - # Apply modification - logger.warning(f"Updating {len(net_interchange_errors.index)} interchanges to new values") - new_areas = cgm_ssh_data.type_tableview('ControlArea').reset_index()[['ID', - 'ControlArea.pTolerance', 'Type']] - new_areas = new_areas.merge(net_interchange_errors[['ControlArea', 'SvPowerFlow.p_post']] - .rename(columns={'ControlArea': 'ID', - 'SvPowerFlow.p_post': 'ControlArea.netInterchange'}), on='ID') - cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(cgm_ssh_data, new_areas) - - return cgm_ssh_data - - -def check_non_boundary_equivalent_injections(cgm_sv_data, - cgm_ssh_data, - original_models, - threshold: float = 0, - fix_errors: bool = False): - """ - Checks equivalent injections that are not on boundary topological nodes - :param cgm_sv_data: merged SV profile - :param cgm_ssh_data: merged SSH profile - :param original_models: igms in triplets - :param threshold: threshold for checking - :param fix_errors: if true then copies values from sv profile to ssh profile - :return cgm_ssh_data - """ - boundary_nodes = original_models.query('KEY == "TopologicalNode.boundaryPoint" & VALUE == "true"')[['ID']] - terminals = (original_models.type_tableview('Terminal').rename_axis('SvPowerFlow.Terminal').reset_index() - .merge(boundary_nodes.rename(columns={'ID': 'Terminal.TopologicalNode'}), - on='Terminal.TopologicalNode', how='outer', indicator=True))[['SvPowerFlow.Terminal', - 'Terminal.ConductingEquipment', - '_merge']] - terminals = terminals[terminals['_merge'] == 'left_only'][['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] - return check_all_kind_of_injections(cgm_sv_data=cgm_sv_data, - cgm_ssh_data=cgm_ssh_data, - original_models=original_models, - injection_name='EquivalentInjection', - fields_to_check={'SvPowerFlow.p': 'EquivalentInjection.p'}, - threshold=threshold, - terminals=terminals, - fix_errors=fix_errors) - - -def check_all_kind_of_injections(cgm_sv_data, - cgm_ssh_data, - original_models, - injection_name: str = 'ExternalNetworkInjection', - fields_to_check: dict = None, - fix_errors: bool = False, - threshold: float = 0, - terminals: pd.DataFrame = None, - report_sum: bool = True): - """ - Compares the given cgm ssh injection values to the corresponding sv powerflow values in cgm sv - :param cgm_sv_data: merged SV profile - :param cgm_ssh_data: merged SSH profile - :param original_models: igms in triplets - :param injection_name: name of the injection - :param fields_to_check: dictionary where key is the field in powerflow and value is the field in injection - :param fix_errors: if true then copies values from sv profile to ssh profile - :param threshold: max allowed mismatch - :param terminals: optional, can give dataframe of terminals as input - :param report_sum: if true prints sum of injections and powerflows to console - :return cgm_ssh_data - """ - if not fields_to_check: - return cgm_ssh_data - - fixed_fields = ['ID'] - try: - original_injections = original_models.type_tableview(injection_name).reset_index() - injections = cgm_ssh_data.type_tableview(injection_name).reset_index() - except AttributeError: - logger.info(f"SSH profile doesn't contain data about {injection_name}") - return cgm_ssh_data - try: - injections_reduced = injections[[*fixed_fields, *fields_to_check.values()]] - original_injections_reduced = original_injections[[*fixed_fields, *fields_to_check.values()]] - except KeyError as ke: - logger.info(f"{injection_name} tableview got error: {ke}") - return cgm_ssh_data - injections_reduced = injections_reduced.merge(original_injections_reduced, on='ID', suffixes=('', '_org')) - if terminals is None: - terminals = (original_models.type_tableview('Terminal') - .rename_axis('SvPowerFlow.Terminal') - .reset_index())[['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] - flows = (cgm_sv_data.type_tableview('SvPowerFlow') - .reset_index())[[*['SvPowerFlow.Terminal'], *fields_to_check.keys()]] - terminals = terminals.merge(flows, on='SvPowerFlow.Terminal') - terminals = terminals.merge(injections_reduced, left_on='Terminal.ConductingEquipment', right_on='ID') - - filtered_list = [] - for flow_field, injection_field in fields_to_check.items(): - filtered_list.append(terminals[abs(terminals[injection_field] - terminals[flow_field]) > threshold]) - if report_sum: - logger.info(f"IGM {injection_field} = {terminals[injection_field + '_org'].sum()} vs " - f"CGM {injection_field} = {terminals[injection_field].sum()} vs " - f"CGM {flow_field} = {terminals[flow_field].sum()}") - if not filtered_list: - return cgm_ssh_data - - filtered = pd.concat(filtered_list).drop_duplicates().reset_index(drop=True) - if not filtered.empty: - logger.warning(f"Found {len(filtered.index)} mismatches between {injection_name} and flow values on terminals") - # Apply modification - if fix_errors: - logger.info(f"Updating {injection_name} values from terminal flow values") - injections_update = injections.merge(filtered[[*fixed_fields, *fields_to_check.keys()]]) - injections_update = injections_update.drop(columns=fields_to_check.values()) - injections_update = injections_update.rename(columns=fields_to_check) - cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(data=cgm_ssh_data, - tableview=injections_update, - update=True, - add=False) - return cgm_ssh_data - - -def run_post_merge_processing(input_models: list, - exported_model: bytes, - opdm_object_meta: dict, - enable_temp_fixes: bool, - task_properties: dict = None, - ): - - # Load original input models to triplets - input_models_triplets = load_opdm_objects_to_triplets(opdm_objects=input_models) - - # Apply corrections to SV profile - sv_data = update_merged_model_sv(sv_data=exported_model, opdm_object_meta=opdm_object_meta) - - # Create update SSH - sv_data, ssh_data, opdm_object_meta = create_updated_ssh(models_as_triplets=input_models_triplets, - input_models = input_models, - sv_data=sv_data, - opdm_object_meta=opdm_object_meta) - # Run temporary modifications on exported model - # Temporary fixes are applied to SV and SSH profiles - if enable_temp_fixes: - # TODO need to revise constantly - sv_data = temporary.remove_equivalent_shunt_section(sv_data, input_models_triplets) - sv_data = temporary.add_missing_sv_tap_steps(sv_data, ssh_data) - sv_data = temporary.remove_small_islands(sv_data, int(SMALL_ISLAND_SIZE)) - sv_data = temporary.remove_duplicate_sv_voltages(cgm_sv_data=sv_data, original_data=input_models_triplets) - sv_data = temporary.check_and_fix_dependencies(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=input_models_triplets) - # TODO following SSH profile fix should be removed once pypowsybl SSH export will be used - ssh_data = temporary.set_paired_boundary_injections_to_zero(original_models=input_models_triplets, - cgm_ssh_data=ssh_data) - - # Run injections check and apply modification if defined in configuration - injection_threshold = float(INJECTION_THRESHOLD) - fix_injection_errors = json.loads(str(FIX_INJECTION_ERRORS).lower()) - - ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, - cgm_sv_data=sv_data, - original_models=input_models_triplets, - injection_name='EnergySource', - threshold=injection_threshold, - fields_to_check={'SvPowerFlow.p': 'EnergySource.activePower'}, - fix_errors=fix_injection_errors) - ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, - cgm_sv_data=sv_data, - original_models=input_models_triplets, - injection_name='ExternalNetworkInjection', - fields_to_check={'SvPowerFlow.p': 'ExternalNetworkInjection.p'}, - threshold=injection_threshold, - fix_errors=fix_injection_errors) - ssh_data = check_non_boundary_equivalent_injections(cgm_sv_data=sv_data, - cgm_ssh_data=ssh_data, - original_models=input_models_triplets, - threshold=injection_threshold, - fix_errors=fix_injection_errors) - - try: - ssh_data = check_net_interchanges(cgm_sv_data=sv_data, - cgm_ssh_data=ssh_data, - original_models=input_models_triplets) - except KeyError: - logger.warning(f"No fields for net interchange correction") - - return sv_data, ssh_data, opdm_object_meta - - -def lvl8_report_cgm(merge_report: dict): - - # Create root - qa_attribs = { - 'created': datetime.datetime.strptime(merge_report["@timestamp"], '%Y-%m-%dT%H:%M:%S.%f').strftime('%Y-%m-%dT%H:%M:%SZ'), - 'schemeVersion': "2.0", - 'serviceProvider': merge_report["merge_entity"], - 'xmlns': "http://entsoe.eu/checks" - } - qa_root = ET.Element("QAReport", attrib=qa_attribs) - - # Add RuleViolations - violations_list = [ - { - 'ruleId': "CGMConvergence", - 'validationLevel': "8", - 'severity': "WARNING", - 'Message': "Power flow could not be calculated for CGM with default settings." - }, - { - 'ruleId': "CGMConvergenceRelaxed", - 'validationLevel': "8", - 'severity': "ERROR", - 'Message': "Power flow could not be calculated for CGM with EU_RELAXED settings." - }, - { - 'ruleId': "CGMConvergenceRelaxed", - 'validationLevel': "8", - 'severity': "ERROR", - 'Message': "Error on Scaling" - } - ] - # TODO:pick the correct setting based on retruned LF setting and convergance from model. Set model quality indicator based on violations - violations = list() - if merge_report["loadflow_status"] == 'CONVERGED': - if merge_report["loadflow_settings"] == 'EU_DEFAULT': - logger.info(f"Merge successful with default settings included in lvl8 report") - quality_indicator_cgm = "Valid" - else: - violations.append(violations_list[0]) - quality_indicator_cgm = "Warning - non fatal inconsistencies" - else: - violations.append(violations_list[1]) - quality_indicator_cgm = "Invalid - inconsistent data" - - #if scaling is failed then set error from error list - if not merge_report['scaled']: - violations.append(violations_list[2]) - quality_indicator_cgm="Invalid - inconsistent data" - - # Create - cgm_attribs = { - 'created': datetime.datetime.strptime(merge_report["@timestamp"], '%Y-%m-%dT%H:%M:%S.%f').strftime('%Y-%m-%dT%H:%M:%SZ'), - 'resource': merge_report['network_meta']['fullModel_ID'], # TODO get here correct content ID - 'scenarioTime': datetime.datetime.fromisoformat(merge_report["@scenario_timestamp"]).strftime('%Y-%m-%dT%H:%M:%SZ'), - 'version': str(merge_report["@version"]), - 'processType': merge_report["time_horizon_id"] if merge_report["@time_horizon"] == 'ID' else merge_report["@time_horizon"], - 'qualityIndicator': quality_indicator_cgm - } - cgm = ET.SubElement(qa_root, "CGM", attrib=cgm_attribs) - - try: - for v in violations: - rv = ET.SubElement(cgm, "RuleViolation", { - 'ruleId': v['ruleId'], - 'validationLevel': v['validationLevel'], - 'severity': v['severity'] - }) - msg = ET.SubElement(rv, "Message") - msg.text = v['Message'] - except: - logger.info(f"No violations present in merge") - - # TODO:pick the TSOs from QA report. Missing parameters below for all IGMs - for i in merge_report['merge_included_entity'] + merge_report['replaced_entity']: - igm = ET.SubElement(cgm, "IGM", { - 'created': i["creation_timestamp"], - 'scenarioTime': datetime.datetime.fromisoformat(i['scenario_timestamp']).strftime('%Y-%m-%dT%H:%M:%SZ'), - 'tso': i['tso'], - 'version': str(i['version']), - 'processType': i['time_horizon'], - 'qualityIndicator': i['quality_indicator'], - }) - resource_igm = ET.SubElement(igm, "resource") - resource_igm.text = i['model_sv_id'] - - # Add EMFInformation - ET.SubElement(cgm, "EMFInformation", { - 'mergingEntity': merge_report["merge_entity"], - 'cgmType': merge_report["merge_type"] - }) - - # Generate final XML - qa_report_lvl8 = ET.tostring(qa_root, encoding='utf-8', xml_declaration=True) - - return qa_report_lvl8 - - -if __name__ == "__main__": - - from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download - from emf.common.helpers.loadflow import load_network_model - from emf.common.loadflow_tool import loadflow_settings - - logging.basicConfig( - format='%(levelname)-10s %(asctime)s.%(msecs)03d %(name)-30s %(funcName)-35s %(lineno)-5d: %(message)s', - datefmt='%Y-%m-%dT%H:%M:%S', - level=logging.INFO, - handlers=[logging.StreamHandler(sys.stdout)] - ) - - time_horizon = '1D' - scenario_date = "2024-05-22T11:30" - merging_area = "EU" - merging_entity = "BALTICRSC" - mas = 'http://www.baltic-rsc.eu/OperationalPlanning' - version = "104" - - valid_models = get_latest_models_and_download(time_horizon, scenario_date, valid=True) - latest_boundary = get_latest_boundary() - - merged_model = load_network_model(valid_models + [latest_boundary]) - solved_model = pypowsybl.loadflow.run_ac(merged_model, loadflow_settings=loadflow_settings.CGM_DEFAULT) - - # Export to OPDM - from emf.common.integrations.opdm import OPDM - - opdm_client = OPDM() - publication_responses = [] - for instance_file in serialized_data: - logger.info(f"Publishing {instance_file.name} to OPDM") - publication_response = opdm_client.publication_request(instance_file, "CGMES") - - publication_responses.append( - {"name": instance_file.name, - "response": publication_response} - ) +import json +import math +import numpy as np +from xml.sax.expatreader import version +import pandas as pd +import pypowsybl +import logging +import sys +import datetime +import triplets +import uuid +import config +import xml.etree.ElementTree as ET +from emf.common.config_parser import parse_app_properties +from emf.common.integrations import elastic +from emf.model_merger import temporary +from emf.common.helpers.time import parse_datetime +from emf.common.helpers.loadflow import get_model_outages, get_network_elements +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, filename_from_opdm_metadata +from emf.common.helpers.utils import sanitize_nan + + +logger = logging.getLogger(__name__) + +parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.post_processing) + + +def is_valid_uuid(uuid_value): + """ + Checks if input is uuid value + For merged SV profile the output uuid can be combination of several existing uuids + :param uuid_value: input value + :return + """ + try: + uuid.UUID(str(uuid_value)) + return True + except ValueError: + return False + + +def export_merged_model(network: pypowsybl.network, + opdm_object_meta: dict, + profiles: list[str] | None = None, + cgm_convention: bool = True, + ): + + # Define which profiles to export + if profiles: + profiles = ",".join(profiles) + else: + profiles = "SV,SSH,TP,EQ" + + # Define whether export using CGM official structure + cgm_export_flag = "False" + if cgm_convention: + cgm_export_flag = "True" + + # Define base name for exported files + file_base_name = filename_from_opdm_metadata(metadata=opdm_object_meta) + + # Define CGMES export parameters + parameters = { + "iidm.export.cgmes.modeling-authority-set": opdm_object_meta['pmd:modelingAuthoritySet'], + "iidm.export.cgmes.base-name": file_base_name, + "iidm.export.cgmes.profiles": profiles, + # For missing instances like "SupplyStation" + "iidm.export.cgmes.topology-kind": 'NODE_BREAKER', + # cgmes-fix-all-invalid-ids fixes non-standard uuid's. Can cause danglingReference errors + # "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids", # identity, cgmes, cgmes-fix-all-invalid-ids + "iidm.export.cgmes.export-sv-injections-for-slacks": "False", + # False sets all boundary flows to zero causing Kirchhoff 1st law and SvPowerFlowBranchInstances2 errors + # "iidm.export.cgmes.export-boundary-power-flows": "False", + "iidm.export.cgmes.cgm_export": cgm_export_flag, + } + + # Export to bytes object + bytes_object = network.save_to_binary_buffer(format="CGMES", parameters=parameters) + bytes_object.name = f"{file_base_name}_{uuid.uuid4()}.zip" + logger.info(f"Exported merged model to {bytes_object.name}") + + # TODO set correct naming of exported files + # a = triplets.rdf_parser.find_all_xml([bytes_object]) + + return bytes_object + + +def create_merged_model_opdm_object(object_id: str, + time_horizon: str, + merging_entity: str, + merging_area: str, + scenario_date: datetime.datetime, + mas: str, + version: str = "001", + profile: str = "SV", + content_type: str = "CGMES", + ): + opdm_object_meta = { + 'opde:Object-Type': 'CGM', + 'pmd:fullModel_ID': object_id, + 'pmd:creationDate': f"{datetime.datetime.now(datetime.UTC):%Y-%m-%dT%H:%M:%S.%fZ}", + 'pmd:timeHorizon': time_horizon, + 'pmd:cgmesProfile': profile, + 'pmd:contentType': content_type, + 'pmd:modelPartReference': "-".join([merging_entity, merging_area]), + 'pmd:mergingEntity': merging_entity, + 'pmd:Area': merging_area, + 'pmd:validFrom': f"{parse_datetime(scenario_date):%Y%m%dT%H%MZ}", + 'pmd:modelingAuthoritySet': mas, + "pmd:isFullModel": "true", + 'pmd:scenarioDate': f"{parse_datetime(scenario_date):%Y-%m-%dT%H:%M:00Z}", + 'pmd:modelid': object_id, + 'pmd:description': f""" + {time_horizon} + pypowsybl_{pypowsybl.__version__} + {merging_entity} + Model: Simplification of reality for given need. + """, + 'pmd:versionNumber': f"{int(version):03d}", + } + + return opdm_object_meta + + +def update_header_from_opdm_object(data: pd.DataFrame, opdm_object: dict): + return triplets.cgmes_tools.update_FullModel_from_dict(data, metadata={ + "Model.version": f"{int(opdm_object['pmd:versionNumber']):03d}", + "Model.created": f"{parse_datetime(opdm_object['pmd:creationDate']):%Y-%m-%dT%H:%M:%S.%fZ}", + "Model.mergingEntity": opdm_object['pmd:mergingEntity'], + "Model.domain": opdm_object['pmd:Area'], + "Model.scenarioTime": f"{parse_datetime(opdm_object['pmd:scenarioDate']):%Y-%m-%dT%H:%M:00Z}", + "Model.description": opdm_object['pmd:description'], + "Model.processType": opdm_object['pmd:timeHorizon'], + }) + + +def update_merged_model_sv(sv_data: bytes, opdm_object_meta: dict): + + # Load SV profile data + sv_data = pd.read_RDF([sv_data]) + + # Update rdfxml header from opdm object metadata + sv_data = update_header_from_opdm_object(data=sv_data, opdm_object=opdm_object_meta) + + # Update file name at 'label' key + sv_data.set_VALUE_at_KEY(key='label', value=filename_from_opdm_metadata(opdm_object_meta, file_type="xml")) + + sv_data = triplets.cgmes_tools.update_FullModel_from_filename(sv_data) + + # Check and fix SV id if necessary + updated_sv_id_map = {} + for old_id in sv_data.query("KEY == 'Type' and VALUE == 'FullModel'").ID.unique(): + if not is_valid_uuid(old_id): + new_id = str(uuid.uuid4()) + updated_sv_id_map[old_id] = new_id + logger.warning(f"SV profile id {old_id} is not valid, assigning: {new_id}") + sv_data = sv_data.replace(updated_sv_id_map) + + return sv_data + + +def load_ssh(input_data: pd.DataFrame | list): + """ + Loads in ssh profiles from list of profiles or takes the slice from dataframe + :param input_data: list of profiles or dataframe + :return dataframe of ssh data + """ + if not isinstance(input_data, pd.DataFrame): + ssh_data = load_opdm_objects_to_triplets(input_data, "SSH") + else: + ssh_files = (input_data[(input_data['KEY'] == 'label') & + (input_data['VALUE'].str.upper().str.contains('SSH'))][['INSTANCE_ID']] + .drop_duplicates()) + ssh_data = input_data.merge(ssh_files, on='INSTANCE_ID') + ssh_data = triplets.cgmes_tools.update_FullModel_from_filename(ssh_data) + return ssh_data + + +def create_updated_ssh(models_as_triplets: pd.DataFrame | list, + sv_data: pd.DataFrame, + opdm_object_meta: dict, + input_models: list = None, + ): + # TODO rewrite to use pypowsybl exported SSH + + ### SSH ## + + # Load original SSH data to created updated SSH + ssh_file_data = input_models or models_as_triplets + ssh_data = load_ssh(ssh_file_data) + + # Update SSH Model.scenarioTime + ssh_data.set_VALUE_at_KEY('Model.scenarioTime', opdm_object_meta['pmd:scenarioDate']) + + # Load full original data to fix issues + # data = load_opdm_data(original_models) + # terminals = data.type_tableview("Terminal") + + # Update SSH data from SV + ssh_update_map = [ + { + "from_class": "SvPowerFlow", + "from_ID": "Terminal.ConductingEquipment", + "from_attribute": "SvPowerFlow.p", + "to_attribute": "EnergyConsumer.p", + }, + { + "from_class": "SvPowerFlow", + "from_ID": "Terminal.ConductingEquipment", + "from_attribute": "SvPowerFlow.q", + "to_attribute": "EnergyConsumer.q", + }, + { + "from_class": "SvPowerFlow", + "from_ID": "Terminal.ConductingEquipment", + "from_attribute": "SvPowerFlow.p", + "to_attribute": "RotatingMachine.p", + }, + { + "from_class": "SvPowerFlow", + "from_ID": "Terminal.ConductingEquipment", + "from_attribute": "SvPowerFlow.q", + "to_attribute": "RotatingMachine.q", + }, + { + "from_class": "SvTapStep", + "from_ID": "SvTapStep.TapChanger", + "from_attribute": "SvTapStep.position", + "to_attribute": "TapChanger.step", + }, + { + "from_class": "SvShuntCompensatorSections", + "from_ID": "SvShuntCompensatorSections.ShuntCompensator", + "from_attribute": "SvShuntCompensatorSections.sections", + "to_attribute": "ShuntCompensator.sections", + }, + { + "from_class": "SvPowerFlow", + "from_ID": "Terminal.ConductingEquipment", + "from_attribute": "SvPowerFlow.p", + "to_attribute": "EquivalentInjection.p" + }, + { + "from_class": "SvPowerFlow", + "from_ID": "Terminal.ConductingEquipment", + "from_attribute": "SvPowerFlow.q", + "to_attribute": "EquivalentInjection.q" + } + ] + # Load terminal from original data + terminals = models_as_triplets.type_tableview("Terminal") + + # Update + for update in ssh_update_map: + # logger.info(f"Updating: {update['from_attribute']} -> {update['to_attribute']}") + source_data = sv_data.type_tableview(update['from_class']).reset_index(drop=True) + + # Merge with terminal, if needed + if terminal_reference := [column_name if ".Terminal" in column_name else None for column_name in source_data.columns][0]: + source_data = source_data.merge(terminals, left_on=terminal_reference, right_on='ID') + logger.debug(f"Added Terminals to {update['from_class']}") + + ssh_data = ssh_data.update_triplet_from_triplet(source_data.rename(columns={ + update['from_ID']: 'ID', + update['from_attribute']: update['to_attribute']} + )[['ID', update['to_attribute']]].set_index('ID').tableview_to_triplet(), add=False) + + # Generate new UUID for updated SSH + updated_ssh_id_map = {} + for OLD_ID in ssh_data.query("KEY == 'Type' and VALUE == 'FullModel'").ID.unique(): + NEW_ID = str(uuid.uuid4()) + updated_ssh_id_map[OLD_ID] = NEW_ID + logger.info(f"Assigned new UUID for updated SSH: {OLD_ID} -> {NEW_ID}") + + # Update SSH ID-s + ssh_data = ssh_data.replace(updated_ssh_id_map) + + # Update in SV SSH references + sv_data = sv_data.replace(updated_ssh_id_map) + + # Add SSH supersedes reference to old SSH + ssh_supersedes_data = pd.DataFrame([{"ID": item[1], "KEY": "Model.Supersedes", "VALUE": item[0]} for item in updated_ssh_id_map.items()]) + ssh_supersedes_data['INSTANCE_ID'] = ssh_data.query("KEY == 'Type'").merge(ssh_supersedes_data.ID)['INSTANCE_ID'] + ssh_data = ssh_data.update_triplet_from_triplet(ssh_supersedes_data) + + # Update SSH metadata + ssh_data = update_header_from_opdm_object(ssh_data, opdm_object_meta) + + # Update SSH filenames + filename_mask = "{scenarioTime:%Y%m%dT%H%MZ}_{processType}_{mergingEntity}-{domain}-{forEntity}_{messageType}_{version:03d}" + ssh_data = triplets.cgmes_tools.update_filename_from_FullModel(ssh_data, filename_mask=filename_mask) + + return sv_data, ssh_data, opdm_object_meta + + +def ensure_paired_equivalent_injection_compatibility(network: pypowsybl.network): + """Where there are paired boundary points, equivalent injections need to be modified to comply + LEVEL7 rule PairedEICompatibility + + Set P and Q to 0 - so that no additional consumption or production is on tie line + """ + logger.info("Configuring paired boundary points equivalent injections: p0/q0 = 0.0") + dangling_lines = network.get_dangling_lines(all_attributes=True) + paired_dangling_lines = dangling_lines[dangling_lines['paired'] == True] + if paired_dangling_lines.empty: + logger.warning(f"No paired dangling lines found in network model") + return network + + # Set p0/q0 to 0 for all paired dangling lines + _updated_p0 = pd.Series(0, index=paired_dangling_lines.index) + _updated_q0 = pd.Series(0, index=paired_dangling_lines.index) + network.update_dangling_lines(id=paired_dangling_lines.index, p0=_updated_p0, q0=_updated_q0) + + return network + + +def ensure_paired_boundary_line_connectivity(network: pypowsybl.network): + logger.info("Aligning paired boundary lines connection status") + dangling_lines = network.get_dangling_lines(all_attributes=True) + # Add cim:Tieflow attribute to dangling lines + dangling_lines['isTieflow'] = dangling_lines.index.isin(network.get_areas_boundaries()["element"]) + paired_dangling_lines = dangling_lines[dangling_lines['paired'] == True] + if paired_dangling_lines.empty: + logger.warning(f"No paired dangling lines found in network model") + return network + + # Identify dangling line pairs where the 'connected' status is inconsistent within each pairing_key group + group = paired_dangling_lines.groupby('pairing_key') + mask_connected = group['connected'].transform(lambda s: s.nunique() > 1) + mask_tieflow = group['isTieflow'].transform( lambda s: s.nunique() > 1) + + mismatched_dangling_lines_con = paired_dangling_lines[mask_connected] + logger.info(f"Boundary lines with non-matching connection status: {mismatched_dangling_lines_con['pairing_key'].unique().tolist()}") + + mismatched_dangling_lines_tie = paired_dangling_lines[mask_tieflow] + logger.info(f"Boundary lines with non-matching cim:Tieflow: {mismatched_dangling_lines_tie['pairing_key'].unique().tolist()}") + + mismatched_dangling_lines = pd.concat([mismatched_dangling_lines_con, mismatched_dangling_lines_tie]) + + # Set all mismatched lines to disconnected (False) + _connected = pd.Series(data=False, index=mismatched_dangling_lines.index) + network.update_dangling_lines(id=mismatched_dangling_lines.index, connected=_connected) + + # Log each change + for i, row in mismatched_dangling_lines.iterrows(): + logger.info(f"Changed status of dangling line {row['name']}: {row['connected']} -> False") + + return network + + +def generate_merge_report(merged_model: object, task: dict): + """ + Creates JSON type report of pypowsybl loadflow results + + Args: + merged_model: merged pypowsybl network + task: task object dict + Returns: + dict: report of merge results + """ + report = merged_model.__dict__ + + # Pop out pypowsybl network + network = report.pop('network') + + # Include task data + report.update({'@timestamp': task.get('@timestamp'), + '@process_id': task.get('process_id'), + '@run_id': task.get('run_id'), + '@job_id': task.get('job_id'), + '@task_id': task.get('@id'), + '@time_horizon': task['task_properties'].get('time_horizon'), + '@scenario_timestamp': task['task_properties'].get('timestamp_utc'), + '@version': int(task['task_properties'].get('version')), + 'merge_type': task['task_properties'].get('merge_type'), + 'merge_entity': task['task_properties'].get('merging_entity'), + }) + + # Include buses count in each component + buses = get_network_elements(network, pypowsybl.network.ElementType.BUS) + buses_by_component = buses.connected_component.value_counts() + for component in report['loadflow']: + component['buses'] = buses_by_component.to_dict().get(component['connected_component_num']) + + # Count network components/islands + report['component_count'] = len(report['loadflow']) + + # Set trustability tag + report.update(evaluate_trustability(report, task['task_properties'])) + + return sanitize_nan(report) + + +def evaluate_trustability(report, properties) -> dict: + + reason = None + if properties["merge_type"] == "BA": + # Evaluate model trustability based on defined config and report keys + report_keys = ['scaled', 'replaced', 'outages'] + property_keys = ['scaling', 'replacement'] + + # Inline logic functions + key_true = lambda key: lambda d: bool(d.get(key)) + all_ = lambda *rules: lambda d: all(rule(d) for rule in rules) + all_none = lambda *keys, exclude=None: lambda d: all(d.get(k) is None for k in keys if k != exclude) + + # Compose conditions + config_all_true = all_(*(key_true(k) for k in property_keys)) + success_all_true = all_(*(key_true(k) for k in report_keys)) + success_all_none = all_none(*report_keys, exclude='scaled') # Scaling is never in None state + + # Evaluate logic + config_enabled = config_all_true(properties) + success_all_true = success_all_true(report) + success_all_none = success_all_none(report) + scaled_correctly = report['scaled'] + + reason_map = { + "scaled": "scaling failed", + "replaced": "replacement failed", + "outages": "outage fixing failed", + } + + # Decide trust level + if config_enabled and success_all_none and scaled_correctly: + trustability = "trusted" + elif config_enabled and success_all_true: + trustability = "semi-trusted" + else: + trustability = "untrusted" + if not config_enabled: + reason = "config is disabled" + else: + # From reason map get correct reason + for key, value in report.items(): + if key in reason_map and not report[key]: + reason = reason_map[key] + else: + trustability = 'not_evaluated' + + return {"trustability": trustability, "untrustability_reason": reason} + +def filter_models(models: list, included_models: list | str = None, excluded_models: list | str = None, filter_on: str = 'pmd:TSO'): + """ + Filters the list of models to include or to exclude specific tsos if they are given. + If included is defined, excluded is not used + :param models: list of igm models + :param included_models: list or string of tso names, if given, only matching models are returned + :param excluded_models: list or string of tso names, if given, matching models will be discarded + :return updated list of igms + """ + + included_models = [included_models] if isinstance(included_models, str) else included_models + excluded_models = [excluded_models] if isinstance(excluded_models, str) else excluded_models + + if included_models: + logger.info(f"Models to be included: {included_models}") + elif excluded_models: + logger.info(f"Models to be excluded: {excluded_models}") + else: + logger.info(f"Including all available models: {[model['pmd:TSO'] for model in models]}") + return models + + filtered_models = [] + + for model in models: + + if included_models: + if model[filter_on] not in included_models: + logger.info(f"Excluded {model[filter_on]}") + continue + + elif excluded_models: + if model[filter_on] in excluded_models: + logger.info(f"Excluded {model[filter_on]}") + continue + + logger.info(f"Included {model[filter_on]}") + filtered_models.append(model) + + return filtered_models + +def filter_tsos(tsos: list, included_models: list | str = None, excluded_models: list | str = None): + """ + Does the exact same thing as the 'filter_models' function above, just the input is a TSO list instead of + a list of already queried model dicts, therefore the filter_on input is not needed. + Note that while "filter_models" returns a list of model metadata dicts, this returns list of strings! + """ + included_models = [included_models] if isinstance(included_models, str) else included_models + excluded_models = [excluded_models] if isinstance(excluded_models, str) else excluded_models + + if included_models: + logger.info(f"Models to be included: {included_models} (pre-metadata-query)") + elif excluded_models: + logger.info(f"Models to be excluded: {excluded_models} (pre-metadata-query)") + else: + logger.info(f"Including all available models: {tsos} (pre-metadata-query)") + return tsos + + filtered_tsos = [] + + for tso in tsos: + + if included_models: + if tso not in included_models: + logger.info(f"Excluded {tso} (pre-metadata-query)") + continue + + elif excluded_models: + if tso in excluded_models: + logger.info(f"Excluded {tso} (pre-metadata-query)") + continue + + logger.info(f"Included {tso} (pre-metadata-query)") + filtered_tsos.append(tso) + + return filtered_tsos + +def filter_models_by_acnp(models: list, merged_model, acnp_dict, acnp_threshold, conform_load_factor): + + def is_within_acnp_deadband(model): + tso = model.get('pmd:TSO') + if not tso or tso not in acnp_dict: + logger.error(f"TSO '{tso}' not found in acnp dict, skipping filtering") + return True + acnp = acnp_dict[tso] + return abs(float(model['ac_net_position']) - float(acnp)) <= float(acnp_threshold) + + def is_within_conformload_deadband(model): + tso = model.get('pmd:TSO') + if not tso or tso not in acnp_dict: + logger.error(f"TSO '{tso}' not found in acnp dict, skipping filtering") + return True + acnp = acnp_dict[tso] + expected_load = model['sum_conform_load'] * float(conform_load_factor) + return expected_load > abs(float(model['ac_net_position']) - float(acnp)) + + excluded_tso_ids = set() + + # ACNP deadband filter + filtered_models = [model for model in models if is_within_acnp_deadband(model)] + excluded_tsos= [ + {'tso': model['pmd:TSO'], 'reason': 'acnp-outside-schedule-deadband'} + for model in models + if model['pmd:TSO'] not in [fm['pmd:TSO'] for fm in filtered_models] + and model['pmd:TSO'] not in excluded_tso_ids + ] + if excluded_tsos: + excluded_tso_ids.update(model['tso'] for model in excluded_tsos) + logger.warning(f"Exluded TSO due to incorrect schedules: {excluded_tso_ids}") + merged_model.excluded.extend(excluded_tsos) + + # Conformload filter + final_models = [model for model in filtered_models if is_within_conformload_deadband(model)] + excluded_tsos= [ + {'tso': model['pmd:TSO'], 'reason': 'conform-load-outside-schedule-difference'} + for model in filtered_models + if model['pmd:TSO'] not in [fm['pmd:TSO'] for fm in final_models] + and model['pmd:TSO'] not in excluded_tso_ids + ] + if excluded_tsos: + excluded_tso_ids.update(model['tso'] for model in excluded_tsos) + logger.warning(f"Exluded TSO due to incorrect conform load: {excluded_tso_ids}") + merged_model.excluded.extend(excluded_tsos) + + return final_models + + +def filter_replacements_by_acnp(models: pd.DataFrame, acnp_dict, acnp_threshold, conform_load_factor): + + models = models[ + (models['pmd:TSO'].apply(lambda x: x not in acnp_dict)) | + ((models['ac_net_position'] - models['pmd:TSO'].apply(lambda x: acnp_dict.get(x, np.nan))).abs() <= float(acnp_threshold))] + models = models[ + (models['pmd:TSO'].apply(lambda x: x not in acnp_dict)) | + (models['sum_conform_load'] * float(conform_load_factor) > (models['ac_net_position'] - models['pmd:TSO'].apply(lambda x: acnp_dict.get(x, np.nan))).abs())] + + return models + + +def update_model_outages(merged_model: object, tso_list: list, scenario_datetime: str, time_horizon: str): + + area_map = {"LITGRID": "Lithuania", "AST": "Latvia", "ELERING": "Estonia"} + outage_areas = [area_map.get(item, item) for item in tso_list] + + elk_service = elastic.Elastic() + + # Get outage eic-mrid mapping + mrid_map = elk_service.get_docs_by_query(index='config-network*', query={"match_all": {}}, size=10000) + mrid_map['mrid'] = mrid_map['mrid'].str.lstrip('_') + + # Get latest UAP parse date + if time_horizon == 'MO': + merge_type = "Month" + elif time_horizon == 'YR': + merge_type = "Year" + else: + merge_type = "Week" + + body = {"size": 1, "query": {"bool": {"must": [{"match": {"Merge": merge_type}}]}}, + "sort": [{"reportParsedDate": {"order": "desc"}}], "fields": ["reportParsedDate"]} + last_uap_version = elk_service.client.search(index='opc-outages-baltics*', body=body)['hits']['hits'][0]['fields']['reportParsedDate'][0] + + # Query for latest outage UAP + uap_query = {"bool": {"must": [{"match": {"reportParsedDate": f"{last_uap_version}"}}, + {"match": {"Merge": merge_type}}]}} + uap_outages = elk_service.get_docs_by_query(index='opc-outages-baltics*', query=uap_query, size=10000) + + # Filter out incorrect elements + uap_outages['mrid'] = uap_outages['mrid'].replace("None", pd.NA) + uap_outages = uap_outages[uap_outages["asset_type"] != "PROD"] + + # Map missing mrid by eic + lookup = mrid_map.set_index(mrid_map[['eic', 'mrid']].columns[0])[mrid_map[['eic', 'mrid']].columns[1]] + uap_outages.loc[:, 'mrid'] = uap_outages['mrid'].fillna(uap_outages['eic'].map(lookup)) + + unmapped_outages = uap_outages[uap_outages['mrid'].isna()] + # Exception rule for old LitPol element + unmapped_outages = unmapped_outages[unmapped_outages['eic'] != "10T-LT-PL-000037"] + if not unmapped_outages.empty: + logger.warning(f"Unable to map following outage mRIDs: {unmapped_outages['name'].values}") + + # Filter outages according to model scenario date and replaced area + filtered_outages = uap_outages[(uap_outages['start_date'] <= scenario_datetime) & (uap_outages['end_date'] >= scenario_datetime)] + filtered_outages = filtered_outages[filtered_outages['Area'].isin(outage_areas)] + mapped_outages = filtered_outages[~filtered_outages['mrid'].isna()] + + # Get disconnected elements in network model + model_outages = pd.DataFrame(get_model_outages(network=merged_model.network)) + mapped_model_outages = pd.merge(model_outages, mrid_map, left_on='grid_id', right_on='mrid', how='inner') + model_area_map = {"LITGRID": "LT", "AST": "LV", "ELERING": "EE"} + model_outage_areas = [model_area_map.get(item, item) for item in tso_list] + filtered_model_outages = mapped_model_outages[mapped_model_outages['country'].isin(model_outage_areas)] + + # Include cross-border lines for reconnection (both dangling lines) + dangling_lines = get_network_elements(network=merged_model.network, + element_type=pypowsybl.network.ElementType.DANGLING_LINE).reset_index(names=['grid_id']) + border_lines = dangling_lines[dangling_lines['pairing_key'].isin(model_outages['pairing_key'])] + relevant_border_lines = border_lines[border_lines['country'].isin(model_outage_areas)] + # Removing any BRELL lines + relevant_border_lines = relevant_border_lines[~relevant_border_lines['lineEnergyIdentificationCodeEIC'].str.contains('RU')] + additional_dangling_lines = dangling_lines[dangling_lines['pairing_key'].isin(relevant_border_lines['pairing_key'])] + + # Merged dataframe of network elements to be reconnected + filtered_model_outages = pd.concat([filtered_model_outages, additional_dangling_lines]).drop_duplicates(subset='grid_id') + filtered_model_outages = filtered_model_outages.where(pd.notnull(filtered_model_outages), None) + + # rename columns + filtered_model_outages = filtered_model_outages.copy()[['name', 'grid_id', 'eic']].rename(columns={'grid_id': 'mrid'}) + mapped_outages = mapped_outages[['name', 'mrid', 'eic']].copy() + mapped_outages.loc[:, 'mrid'] = mapped_outages['mrid'].str.lstrip('_') + + logger.info(f"Updating outages in merged model areas: {model_outage_areas}") + + # Reconnecting outages from network-config list + outages_updated = {} + filtered_model_outages["eic"] = (filtered_model_outages["eic"].astype(object).where(filtered_model_outages["eic"].notna(), None)) + for index, outage in filtered_model_outages.iterrows(): + try: + if merged_model.network.connect(outage['mrid']): + logger.info(f"Successfully reconnected: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages = True + outage_dict = outage.to_dict() + outage_dict.update({'status': 'connected'}) + outages_updated[outage_dict['mrid']] = outage_dict + else: + if uap_outages['grid_id'].str.contains(outage['mrid']).any(): + logger.info(f"Element is already connected: {outage['name']} [mrid: {outage['mrid']}]") + else: + logger.error(f"Failed to connect element: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + except Exception as e: + logger.error((e, outage['name'])) + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + merged_model.outages = False + continue + + # Applying outages from UAP + mapped_outages["eic"] = (mapped_outages["eic"].astype(object).where(mapped_outages["eic"].notna(), None)) + for index, outage in mapped_outages.iterrows(): + try: + if merged_model.network.disconnect(outage['mrid']): + logger.info(f"Successfully disconnected: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages = True + outage_dict = outage.to_dict() + outage_dict.update({'status': 'disconnected'}) + outages_updated[outage_dict['mrid']] = outage_dict + else: + if uap_outages['mrid'].str.contains(("_" + outage['mrid'])).any(): + logger.info(f"Element is already in outage: {outage['name']} [mrid: {outage['mrid']}]") + else: + logger.error(f"Failed to disconnect element: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + except Exception as e: + logger.error((e, outage['name'], outage['mrid'])) + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + merged_model.outages = False + continue + + # Keep only important keys of updated outages + merged_model.outages_updated = list(outages_updated.values()) + + if merged_model.outages_unmapped: + merged_model.outages = False + + # Sanitise NaN values in merge report + merged_model.outages_updated = [{k: None if isinstance(v, float) and math.isnan(v) else v for k, v in d.items()} + for d in merged_model.outages_updated] + merged_model.outages_unmapped = [{k: None if isinstance(v, float) and math.isnan(v) else v for k, v in d.items()} + for d in merged_model.outages_unmapped] + + return merged_model + + +def set_intraday_time_horizon(scenario_datetime, task_creation_time): + """ + Finds time difference between task creation time and scenario timestamp. Converts it to hours and finds the hour + number corresponding to intraday run (number of hours that scenario timestamp is ahead from task creation time) + Here are multiple ways to calculate (must keep in mind start and end times/dates) + 1) Ceil: f"{math.ceil((_scenario_datetime - _task_creation_time).seconds / 3600):02d}" + 2) Round: f"{int((_scenario_datetime - _task_creation_time).seconds / 3600):02d}" + 3) Floor: f"{math.floor((_scenario_datetime - _task_creation_time).seconds / 3600):02d}" + Take into account date change + 4) Min(max(Ceil)): max(math.ceil((time_diff.days * 24 * 3600 + time_diff.seconds) / 3600), 1) + 5) Min(max(Round)): max(int((time_diff.days * 24 * 3600 + time_diff.seconds) / 3600), 1) + 6) Min(max(Floor)): max(math.floor((time_diff.days * 24 * 3600 + time_diff.seconds) / 3600), 1) + :param scenario_datetime: scenario timestamp for intraday run + :param task_creation_time: timestamp when the task was created + :return: time horizon for intraday run as a string + """ + max_time_horizon_value = 36 + calculated_time_horizon = '01' # DEFAULT VALUE, CHANGE THIS + _task_creation_time = parse_datetime(task_creation_time, keep_timezone=False) + _scenario_datetime = parse_datetime(scenario_datetime, keep_timezone=False) + time_diff = _scenario_datetime - _task_creation_time + if 0 <= time_diff.days <= 1: + time_horizon_actual = math.floor((time_diff.days * 24 * 3600 + time_diff.seconds) / 3600) + # just in case cut it to bigger than 1 once again + time_horizon_actual = max(time_horizon_actual, 1) + if time_horizon_actual <= max_time_horizon_value: + calculated_time_horizon = f"{time_horizon_actual:02d}" + return calculated_time_horizon + + +def check_net_interchanges(cgm_sv_data, cgm_ssh_data, original_models): + """ + An attempt to calculate the net interchange 2 values and check them against those provided in ssh profiles + :param cgm_sv_data: merged sv profile + :param cgm_ssh_data: merged ssh profile + :param original_models: igms in triplets + :param fix_errors: injects new calculated flows into merged ssh profiles + :param threshold: specify threshold if needed + :return (updated) ssh profiles + """ + try: + control_areas = (original_models.type_tableview('ControlArea') + .rename_axis('ControlArea') + .reset_index())[['ControlArea', 'ControlArea.netInterchange', 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', 'IdentifiedObject.name']] + except KeyError: + control_areas = original_models.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + ssh_areas = cgm_ssh_data.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + control_areas = control_areas.merge(ssh_areas, on='ControlArea')[['ControlArea', 'ControlArea.netInterchange', + 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', + 'IdentifiedObject.name']] + tie_flows = (original_models.type_tableview('TieFlow') + .rename_axis('TieFlow').rename(columns={'TieFlow.ControlArea': 'ControlArea', + 'TieFlow.Terminal': 'Terminal'}) + .reset_index())[['ControlArea', 'Terminal', 'TieFlow.positiveFlowIn']] + tie_flows = tie_flows.merge(control_areas[['ControlArea']], on='ControlArea') + try: + terminals = (original_models.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal', 'ACDCTerminal.connected']] + except KeyError: + terminals = (original_models.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal']] + tie_flows = tie_flows.merge(terminals, on='Terminal') + try: + power_flows_pre = (original_models.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + tie_flows = tie_flows.merge(power_flows_pre, on='Terminal', how='left') + except Exception as error: + logger.error(f"Was not able to get tie flows from original models with exception: {error}") + power_flows_post = (cgm_sv_data.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + + tie_flows = tie_flows.merge(power_flows_post, on='Terminal', how='left', + suffixes=('_pre', '_post')) + try: + tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p_pre', 'SvPowerFlow.p_post']] + .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) + .rename_axis('ControlArea').reset_index()) + except KeyError: + tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p']] + .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) + .rename_axis('ControlArea').reset_index()) + tie_flows_grouped = tie_flows_grouped.rename(columns={'SvPowerFlow.p': 'SvPowerFlow.p_post'}) + tie_flows_grouped = control_areas.merge(tie_flows_grouped, on='ControlArea') + + net_interchange_errors = tie_flows_grouped.loc[ + tie_flows_grouped['ControlArea.netInterchange'].ne(tie_flows_grouped['SvPowerFlow.p_post'])] + + if not net_interchange_errors.empty: + # Apply modification + logger.warning(f"Updating {len(net_interchange_errors.index)} interchanges to new values") + new_areas = cgm_ssh_data.type_tableview('ControlArea').reset_index()[['ID', + 'ControlArea.pTolerance', 'Type']] + new_areas = new_areas.merge(net_interchange_errors[['ControlArea', 'SvPowerFlow.p_post']] + .rename(columns={'ControlArea': 'ID', + 'SvPowerFlow.p_post': 'ControlArea.netInterchange'}), on='ID') + cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(cgm_ssh_data, new_areas) + + return cgm_ssh_data + + +def check_non_boundary_equivalent_injections(cgm_sv_data, + cgm_ssh_data, + original_models, + threshold: float = 0, + fix_errors: bool = False): + """ + Checks equivalent injections that are not on boundary topological nodes + :param cgm_sv_data: merged SV profile + :param cgm_ssh_data: merged SSH profile + :param original_models: igms in triplets + :param threshold: threshold for checking + :param fix_errors: if true then copies values from sv profile to ssh profile + :return cgm_ssh_data + """ + boundary_nodes = original_models.query('KEY == "TopologicalNode.boundaryPoint" & VALUE == "true"')[['ID']] + terminals = (original_models.type_tableview('Terminal').rename_axis('SvPowerFlow.Terminal').reset_index() + .merge(boundary_nodes.rename(columns={'ID': 'Terminal.TopologicalNode'}), + on='Terminal.TopologicalNode', how='outer', indicator=True))[['SvPowerFlow.Terminal', + 'Terminal.ConductingEquipment', + '_merge']] + terminals = terminals[terminals['_merge'] == 'left_only'][['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] + return check_all_kind_of_injections(cgm_sv_data=cgm_sv_data, + cgm_ssh_data=cgm_ssh_data, + original_models=original_models, + injection_name='EquivalentInjection', + fields_to_check={'SvPowerFlow.p': 'EquivalentInjection.p'}, + threshold=threshold, + terminals=terminals, + fix_errors=fix_errors) + + +def check_all_kind_of_injections(cgm_sv_data, + cgm_ssh_data, + original_models, + injection_name: str = 'ExternalNetworkInjection', + fields_to_check: dict = None, + fix_errors: bool = False, + threshold: float = 0, + terminals: pd.DataFrame = None, + report_sum: bool = True): + """ + Compares the given cgm ssh injection values to the corresponding sv powerflow values in cgm sv + :param cgm_sv_data: merged SV profile + :param cgm_ssh_data: merged SSH profile + :param original_models: igms in triplets + :param injection_name: name of the injection + :param fields_to_check: dictionary where key is the field in powerflow and value is the field in injection + :param fix_errors: if true then copies values from sv profile to ssh profile + :param threshold: max allowed mismatch + :param terminals: optional, can give dataframe of terminals as input + :param report_sum: if true prints sum of injections and powerflows to console + :return cgm_ssh_data + """ + if not fields_to_check: + return cgm_ssh_data + + fixed_fields = ['ID'] + try: + original_injections = original_models.type_tableview(injection_name).reset_index() + injections = cgm_ssh_data.type_tableview(injection_name).reset_index() + except AttributeError: + logger.info(f"SSH profile doesn't contain data about {injection_name}") + return cgm_ssh_data + try: + injections_reduced = injections[[*fixed_fields, *fields_to_check.values()]] + original_injections_reduced = original_injections[[*fixed_fields, *fields_to_check.values()]] + except KeyError as ke: + logger.info(f"{injection_name} tableview got error: {ke}") + return cgm_ssh_data + injections_reduced = injections_reduced.merge(original_injections_reduced, on='ID', suffixes=('', '_org')) + if terminals is None: + terminals = (original_models.type_tableview('Terminal') + .rename_axis('SvPowerFlow.Terminal') + .reset_index())[['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] + flows = (cgm_sv_data.type_tableview('SvPowerFlow') + .reset_index())[[*['SvPowerFlow.Terminal'], *fields_to_check.keys()]] + terminals = terminals.merge(flows, on='SvPowerFlow.Terminal') + terminals = terminals.merge(injections_reduced, left_on='Terminal.ConductingEquipment', right_on='ID') + + filtered_list = [] + for flow_field, injection_field in fields_to_check.items(): + filtered_list.append(terminals[abs(terminals[injection_field] - terminals[flow_field]) > threshold]) + if report_sum: + logger.info(f"IGM {injection_field} = {terminals[injection_field + '_org'].sum()} vs " + f"CGM {injection_field} = {terminals[injection_field].sum()} vs " + f"CGM {flow_field} = {terminals[flow_field].sum()}") + if not filtered_list: + return cgm_ssh_data + + filtered = pd.concat(filtered_list).drop_duplicates().reset_index(drop=True) + if not filtered.empty: + logger.warning(f"Found {len(filtered.index)} mismatches between {injection_name} and flow values on terminals") + # Apply modification + if fix_errors: + logger.info(f"Updating {injection_name} values from terminal flow values") + injections_update = injections.merge(filtered[[*fixed_fields, *fields_to_check.keys()]]) + injections_update = injections_update.drop(columns=fields_to_check.values()) + injections_update = injections_update.rename(columns=fields_to_check) + cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(data=cgm_ssh_data, + tableview=injections_update, + update=True, + add=False) + return cgm_ssh_data + + +def run_post_merge_processing(input_models: list, + exported_model: bytes, + opdm_object_meta: dict, + enable_temp_fixes: bool, + task_properties: dict = None, + ): + + # Load original input models to triplets + input_models_triplets = load_opdm_objects_to_triplets(opdm_objects=input_models) + + # Apply corrections to SV profile + sv_data = update_merged_model_sv(sv_data=exported_model, opdm_object_meta=opdm_object_meta) + + # Create update SSH + sv_data, ssh_data, opdm_object_meta = create_updated_ssh(models_as_triplets=input_models_triplets, + input_models = input_models, + sv_data=sv_data, + opdm_object_meta=opdm_object_meta) + # Run temporary modifications on exported model + # Temporary fixes are applied to SV and SSH profiles + if enable_temp_fixes: + # TODO need to revise constantly + sv_data = temporary.remove_equivalent_shunt_section(sv_data, input_models_triplets) + sv_data = temporary.add_missing_sv_tap_steps(sv_data, ssh_data) + sv_data = temporary.remove_small_islands(sv_data, int(SMALL_ISLAND_SIZE)) + sv_data = temporary.remove_duplicate_sv_voltages(cgm_sv_data=sv_data, original_data=input_models_triplets) + sv_data = temporary.check_and_fix_dependencies(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=input_models_triplets) + # TODO following SSH profile fix should be removed once pypowsybl SSH export will be used + ssh_data = temporary.set_paired_boundary_injections_to_zero(original_models=input_models_triplets, + cgm_ssh_data=ssh_data) + + # Run injections check and apply modification if defined in configuration + injection_threshold = float(INJECTION_THRESHOLD) + fix_injection_errors = json.loads(str(FIX_INJECTION_ERRORS).lower()) + + ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, + cgm_sv_data=sv_data, + original_models=input_models_triplets, + injection_name='EnergySource', + threshold=injection_threshold, + fields_to_check={'SvPowerFlow.p': 'EnergySource.activePower'}, + fix_errors=fix_injection_errors) + ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, + cgm_sv_data=sv_data, + original_models=input_models_triplets, + injection_name='ExternalNetworkInjection', + fields_to_check={'SvPowerFlow.p': 'ExternalNetworkInjection.p'}, + threshold=injection_threshold, + fix_errors=fix_injection_errors) + ssh_data = check_non_boundary_equivalent_injections(cgm_sv_data=sv_data, + cgm_ssh_data=ssh_data, + original_models=input_models_triplets, + threshold=injection_threshold, + fix_errors=fix_injection_errors) + + try: + ssh_data = check_net_interchanges(cgm_sv_data=sv_data, + cgm_ssh_data=ssh_data, + original_models=input_models_triplets) + except KeyError: + logger.warning(f"No fields for net interchange correction") + + return sv_data, ssh_data, opdm_object_meta + + +def lvl8_report_cgm(merge_report: dict): + + # Create root + qa_attribs = { + 'created': datetime.datetime.strptime(merge_report["@timestamp"], '%Y-%m-%dT%H:%M:%S.%f').strftime('%Y-%m-%dT%H:%M:%SZ'), + 'schemeVersion': "2.0", + 'serviceProvider': merge_report["merge_entity"], + 'xmlns': "http://entsoe.eu/checks" + } + qa_root = ET.Element("QAReport", attrib=qa_attribs) + + # Add RuleViolations + violations_list = [ + { + 'ruleId': "CGMConvergence", + 'validationLevel': "8", + 'severity': "WARNING", + 'Message': "Power flow could not be calculated for CGM with default settings." + }, + { + 'ruleId': "CGMConvergenceRelaxed", + 'validationLevel': "8", + 'severity': "ERROR", + 'Message': "Power flow could not be calculated for CGM with EU_RELAXED settings." + }, + { + 'ruleId': "CGMConvergenceRelaxed", + 'validationLevel': "8", + 'severity': "ERROR", + 'Message': "Error on Scaling" + } + ] + # TODO:pick the correct setting based on retruned LF setting and convergance from model. Set model quality indicator based on violations + violations = list() + if merge_report["loadflow_status"] == 'CONVERGED': + if merge_report["loadflow_settings"] == 'EU_DEFAULT': + logger.info(f"Merge successful with default settings included in lvl8 report") + quality_indicator_cgm = "Valid" + else: + violations.append(violations_list[0]) + quality_indicator_cgm = "Warning - non fatal inconsistencies" + else: + violations.append(violations_list[1]) + quality_indicator_cgm = "Invalid - inconsistent data" + + #if scaling is failed then set error from error list + if not merge_report['scaled']: + violations.append(violations_list[2]) + quality_indicator_cgm="Invalid - inconsistent data" + + # Create + cgm_attribs = { + 'created': datetime.datetime.strptime(merge_report["@timestamp"], '%Y-%m-%dT%H:%M:%S.%f').strftime('%Y-%m-%dT%H:%M:%SZ'), + 'resource': merge_report['network_meta']['fullModel_ID'], # TODO get here correct content ID + 'scenarioTime': datetime.datetime.fromisoformat(merge_report["@scenario_timestamp"]).strftime('%Y-%m-%dT%H:%M:%SZ'), + 'version': str(merge_report["@version"]), + 'processType': merge_report["time_horizon_id"] if merge_report["@time_horizon"] == 'ID' else merge_report["@time_horizon"], + 'qualityIndicator': quality_indicator_cgm + } + cgm = ET.SubElement(qa_root, "CGM", attrib=cgm_attribs) + + try: + for v in violations: + rv = ET.SubElement(cgm, "RuleViolation", { + 'ruleId': v['ruleId'], + 'validationLevel': v['validationLevel'], + 'severity': v['severity'] + }) + msg = ET.SubElement(rv, "Message") + msg.text = v['Message'] + except: + logger.info(f"No violations present in merge") + + # TODO:pick the TSOs from QA report. Missing parameters below for all IGMs + for i in merge_report['merge_included_entity'] + merge_report['replaced_entity']: + igm = ET.SubElement(cgm, "IGM", { + 'created': i["creation_timestamp"], + 'scenarioTime': datetime.datetime.fromisoformat(i['scenario_timestamp']).strftime('%Y-%m-%dT%H:%M:%SZ'), + 'tso': i['tso'], + 'version': str(i['version']), + 'processType': i['time_horizon'], + 'qualityIndicator': i['quality_indicator'], + }) + resource_igm = ET.SubElement(igm, "resource") + resource_igm.text = i['model_sv_id'] + + # Add EMFInformation + ET.SubElement(cgm, "EMFInformation", { + 'mergingEntity': merge_report["merge_entity"], + 'cgmType': merge_report["merge_type"] + }) + + # Generate final XML + qa_report_lvl8 = ET.tostring(qa_root, encoding='utf-8', xml_declaration=True) + + return qa_report_lvl8 + + +if __name__ == "__main__": + + from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download + from emf.common.helpers.loadflow import load_network_model + from emf.common.loadflow_tool import loadflow_settings + + logging.basicConfig( + format='%(levelname)-10s %(asctime)s.%(msecs)03d %(name)-30s %(funcName)-35s %(lineno)-5d: %(message)s', + datefmt='%Y-%m-%dT%H:%M:%S', + level=logging.INFO, + handlers=[logging.StreamHandler(sys.stdout)] + ) + + time_horizon = '1D' + scenario_date = "2024-05-22T11:30" + merging_area = "EU" + merging_entity = "BALTICRSC" + mas = 'http://www.baltic-rsc.eu/OperationalPlanning' + version = "104" + + valid_models = get_latest_models_and_download(time_horizon, scenario_date, valid=True) + latest_boundary = get_latest_boundary() + + merged_model = load_network_model(valid_models + [latest_boundary]) + solved_model = pypowsybl.loadflow.run_ac(merged_model, loadflow_settings=loadflow_settings.CGM_DEFAULT) + + # Export to OPDM + from emf.common.integrations.opdm import OPDM + + opdm_client = OPDM() + publication_responses = [] + for instance_file in serialized_data: + logger.info(f"Publishing {instance_file.name} to OPDM") + publication_response = opdm_client.publication_request(instance_file, "CGMES") + + publication_responses.append( + {"name": instance_file.name, + "response": publication_response} + ) diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 6bb6def5..8bfc2fba 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -1,653 +1,658 @@ -import logging -import pypowsybl -import config -import json -import time -from uuid import uuid4 -import datetime -from dataclasses import dataclass, field -from typing import List -from emf.common.helpers.time import parse_datetime -from io import BytesIO -from zipfile import ZipFile -from emf.common.config_parser import parse_app_properties -from emf.common.integrations import opdm, minio_api, elastic, edx -from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download -from emf.common.integrations.object_storage.schedules import query_acnp_schedules, query_hvdc_schedules, \ - calculate_ac_net_position -from emf.common.loadflow_tool import loadflow_settings, settings_manager -from emf.common.helpers.utils import attr_to_dict, convert_dict_str_to_bool -from emf.common.helpers.cgmes import export_to_cgmes_zip -from emf.common.helpers.opdm_objects import get_opdm_component_data_bytes -from emf.common.helpers.loadflow import load_network_model -from emf.common.helpers.tasks import update_task_status -from emf.model_merger import merge_functions -from emf.model_merger import scaler -from emf.model_merger.merge_functions import filter_models_by_acnp -from emf.model_merger.replacement import run_replacement, get_tsos_available_in_storage -from emf.model_merger.temporary import handle_igm_ssh_vs_cgm_ssh_error -from emf.common.logging.custom_logger import get_elk_logging_handler -from concurrent.futures import ThreadPoolExecutor -from lxml import etree - -logger = logging.getLogger(__name__) -parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.merger) -executor = ThreadPoolExecutor(max_workers=20) - - -def async_call(function, callback=None, *args, **kwargs): - future = executor.submit(function, *args, **kwargs) - if callback: - future.add_done_callback(lambda f: callback(f.result())) - return future - - -def log_opdm_response(response): - logger.debug(etree.tostring(response, pretty_print=True).decode()) - - -@dataclass -class MergedModel: - network: pypowsybl.network = None - time_horizon: str = None - time_horizon_id: str = field(default_factory=str) - name = None - loadflow_status: str | None = None - - # Status flags - scaled: bool = None - replaced: bool = None - outages: bool = None - uploaded_to_opde: bool = False - uploaded_to_minio: bool = False - - # Extended data - loadflow: List = field(default_factory=list) - excluded: List = field(default_factory=list) - scaled_entity: List = field(default_factory=list) - scaled_hvdc: List = field(default_factory=list) - replaced_entity: List = field(default_factory=list) - replacement_reason: List = field(default_factory=list) - outages_updated: List = field(default_factory=list) - outages_unmapped: List = field(default_factory=list) - merge_included_entity: List = field(default_factory=list) - - -@dataclass(init=False) -class ModelEntity: - data_source: str = "OPDM" - quality_indicator: str = "Valid" - tso: str = None - time_horizon: str = None - scenario_timestamp: str = None - model_sv_id: str = None - version: int = None - quality_indicator: str = "Valid" - creation_timestamp: str = None - file_name: str = None - - def __init__(self, data_source: str, quality_indicator: str, **kwargs): - self.data_source = data_source - self.quality_indicator = quality_indicator - self.tso = kwargs.get('pmd:TSO', 'unknown') - self.time_horizon = kwargs.get('pmd:timeHorizon', 'unknown') - self.scenario_timestamp = kwargs.get('pmd:scenarioDate', 'unknown') - self.model_sv_id = kwargs.get('pmd:fullModel_ID', 'unknown') - self.version = int(kwargs.get('pmd:version', 999)) - self.creation_timestamp = kwargs.get('pmd:creationDate', 'unknown') - self.file_name = kwargs.get('pmd:fileName', 'unknown') - - -class HandlerMergeModels: - - def __init__(self): - self.minio_service = minio_api.ObjectStorage() - self.elk_logging_handler = get_elk_logging_handler() - self.opdm_service = None - - @staticmethod - def run_loadflow(merged_model): - # Set starting point of lf settings priority list - if json.loads(ENABLE_DYNAMIC_MERGE_SETTINGS.lower()): - settings_list = [param.strip() for param in MERGE_LOAD_FLOW_SETTINGS_PRIORITY.split(",")] - settings_priority = next((i for i, value in enumerate(settings_list) if value == MERGE_LOAD_FLOW_SETTINGS), - None) - settings_list = settings_list[settings_priority:] - else: - settings_list = [MERGE_LOAD_FLOW_SETTINGS] - - for lf_settings in settings_list: - logger.info(f"Solving loadflow with settings: {lf_settings}") - # report = pypowsybl.report.Reporter() - manager = settings_manager.LoadflowSettingsManager(settings_keyword=lf_settings) - pp_loadflow_parameters = manager.build_pypowsybl_parameters() - result = pypowsybl.loadflow.run_ac(network=merged_model.network, - parameters=pp_loadflow_parameters, - # reporter=loadflow_report, - ) - if result[0].status_text == 'Converged': - break - else: - logger.warning(f"Failed to solve loadflow with settings: {lf_settings}") - - result_dict = [attr_to_dict(island) for island in result] - # Modify all nested objects to native data types - for island in result_dict: - island['status'] = island['status'].name - # Extract only first slack bus from internal pypowsybl object - slack_bus_results = island.pop('slack_bus_results') - if slack_bus_results: - island['slack_bus_id'] = getattr(slack_bus_results[0], 'id', 'undefined') - island['active_power_mismatch'] = getattr(slack_bus_results[0], 'active_power_mismatch', float()) - else: - island['slack_bus_id'] = 'undefined' - island['active_power_mismatch'] = float() - - # merged_model.loadflow = json.loads(loadflow_report.to_json()) - # merged_model.loadflow = str(loadflow_report) - merged_model.loadflow = [island for island in result_dict if island['reference_bus_id']] - merged_model.loadflow_status = result[0].status.name # store main island loadflow status - merged_model.loadflow_settings = lf_settings - - return merged_model, pp_loadflow_parameters - - def handle(self, task_object: dict, properties: dict, **kwargs): - - start_time = datetime.datetime.now(datetime.UTC) - - # Create instance of merged model - merged_model = MergedModel() - - # Parse relevant data from Task - task = task_object - if not isinstance(task, dict): - task = json.loads(task_object) - - # Convert task fields to bool where necessary - task = convert_dict_str_to_bool(task) - - # TODO - make it to a wrapper once it is settled/standardized how this info is exchanged - # Initialize trace - self.elk_logging_handler.start_trace(task) - logger.debug(task) - - # Set task to started - update_task_status(task, "started") - - # Task configuration - task_creation_time = task.get('task_creation_time') - task_properties = task.get('task_properties', {}) - included_models = task_properties.get('included', []) - excluded_models = task_properties.get('excluded', []) - local_import_models = task_properties.get('local_import', []) - replace_tso = task_properties.get('replace_tso', []) - time_horizon = task_properties["time_horizon"] - scenario_datetime = task_properties["timestamp_utc"] - schedule_start = task_properties.get("reference_schedule_start_utc") - schedule_end = task_properties.get("reference_schedule_end_utc") - schedule_time_horizon = task_properties.get("reference_schedule_time_horizon") - merging_area = task_properties["merge_type"] - merging_entity = task_properties["merging_entity"] - mas = task_properties["mas"] - version = task_properties["version"] - model_replacement = task_properties["replacement"] - model_scaling = task_properties["scaling"] - outage_update = task_properties["outage_update"] - force_outage_fix = task_properties['force_outage_fix'] - model_upload_to_opdm = task_properties["upload_to_opdm"] - model_upload_to_minio = task_properties["upload_to_minio"] - model_merge_report_send_to_elk = task_properties["send_merge_report"] - post_temp_fixes = task_properties['post_temp_fixes'] - lvl8_reporting = task_properties['lvl8_reporting'] - - # Get aligned schedules - # Set default time horizon and scenario timestamp if not provided - if not schedule_time_horizon or schedule_time_horizon == "AUTO": - schedule_time_horizon = time_horizon - - if not schedule_start: - schedule_start = scenario_datetime - - ac_schedules = query_acnp_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) - dc_schedules = query_hvdc_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) - acnp_dict = calculate_ac_net_position(ac_schedules) - - # Collect valid models from ObjectStorage - downloaded_models = get_latest_models_and_download(time_horizon=time_horizon, - scenario_date=scenario_datetime, - valid=True, - data_source='OPDM') - latest_boundary = get_latest_boundary() - - # Filter out models that are not to be used in merge - models = merge_functions.filter_models(models=downloaded_models, - included_models=included_models, - excluded_models=excluded_models, - filter_on='pmd:TSO') - - # Get additional models from ObjectStorage if local import is configured - if local_import_models: - additional_models = get_latest_models_and_download(time_horizon=time_horizon, - scenario_date=scenario_datetime, - valid=True, - data_source='PDN') - additional_models = merge_functions.filter_models(models=additional_models, - included_models=local_import_models, - filter_on='pmd:TSO') - - additional_tsos = {model['pmd:TSO'] for model in additional_models} - missing_local_import = [tso for tso in local_import_models if tso not in additional_tsos] - merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-pdn'} for tso in missing_local_import]) - - else: - additional_models = [] - missing_local_import = [] - - # Check missing models for replacement - if included_models: - models_tsos = {model['pmd:TSO'] for model in models} - missing_models = [tso for tso in included_models if tso not in models_tsos] - - if missing_models: - merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-opdm'} for tso in missing_models]) - - # find RMM models: - missing_models_rmm = [tso for tso in missing_models if merging_area == "BA"] - - if missing_models_rmm: - # Get PDN models when OPDM models missing - pdn_auto_models = get_latest_models_and_download(time_horizon=time_horizon, - scenario_date=scenario_datetime, - valid=True, - data_source='PDN') - pdn_auto_models = merge_functions.filter_models(models=pdn_auto_models, - included_models=missing_models_rmm, - filter_on='pmd:TSO') - - # Cache PDN TSO set - pdn_tsos = {m['pmd:TSO'] for m in pdn_auto_models} - missing_pdn_auto = [tso for tso in missing_models_rmm if tso not in pdn_tsos] - - if missing_pdn_auto: - logging.info(f"OPDM and PDN missing for {missing_pdn_auto}") - - replaced_with_pdn = [tso for tso in missing_models_rmm if tso not in missing_pdn_auto] - if replaced_with_pdn: - logging.info(f"OPDM missing for {replaced_with_pdn} - replaced with PDN models") - - models = models + pdn_auto_models - - # Update exclusion list reason - for item in merged_model.excluded: - if item['tso'] in missing_pdn_auto: - item['reason'] = 'missing-opdm-and-pdn' - - # Rewrite missing_models - missing_models = [tso for tso in missing_models if tso not in pdn_tsos] - - else: - if model_replacement: - # Get TSOs which models are available in storage for replacement period - available_tsos = get_tsos_available_in_storage(time_horizon=time_horizon) - valid_model_tsos = [model['pmd:TSO'] for model in models] - # Need to ensure that excluded models by task configuration would not be taken in replacement context - missing_models = [tso for tso in available_tsos if tso not in valid_model_tsos + excluded_models] - if missing_models: - merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-opdm'} for tso in missing_models]) - else: - missing_models = [] - - # Exclude models that are outside scheduled AC net position deadband - if acnp_dict: - logger.info("Excluding models with incorrect ACNP") - filterd_models = filter_models_by_acnp(models, merged_model, acnp_dict, ACNP_THRESHOLD, CONFORM_LOAD_FACTOR) - filterd_additional_models = filter_models_by_acnp(additional_models, merged_model, acnp_dict, ACNP_THRESHOLD, - CONFORM_LOAD_FACTOR) - if included_models: - # Update missing models - missing_models = [m for m in included_models if m not in [m['pmd:TSO'] for m in filterd_models]] - missing_local_import = [m for m in local_import_models if m not in [m['pmd:TSO'] for m in filterd_additional_models]] - models = filterd_models - additional_models = filterd_additional_models - elif model_replacement: - models_tsos = {model['pmd:TSO'] for model in models} - excluded_incorrect = [tso for tso in valid_model_tsos if - tso not in models_tsos and tso not in missing_models] - missing_models = missing_models + excluded_incorrect - - # Execute consolidated model replacement logic - models, additional_models = run_replacement( - models=models, - additional_models=additional_models, - model_replacement=model_replacement, - local_import_models=local_import_models, - missing_local_import=missing_local_import, - missing_models=missing_models, - replace_tso=replace_tso, - time_horizon=time_horizon, - scenario_datetime=scenario_datetime, - merged_model=merged_model, - acnp_dict=acnp_dict, - acnp_threshold=ACNP_THRESHOLD, - conform_load_factor=CONFORM_LOAD_FACTOR - ) - - merged_model.merge_included_entity = [ - ModelEntity(data_source='OPDM', quality_indicator='Valid', **model).__dict__ for model in models] - - merged_model.merge_included_entity.extend( - [ModelEntity(data_source='PDN', quality_indicator='Valid', **model).__dict__ for model in - additional_models]) - - # Store models together with boundary set and check whether there are enough models to merge - input_models = models + additional_models + [latest_boundary] - if len(input_models) < 2: - logger.warning("No valid models found for merging, exiting merge process") - properties.headers['success'] = False - return task_object, properties - - # Load network model and merge - merge_start = datetime.datetime.now(datetime.UTC) - merged_model.network = load_network_model(opdm_objects=input_models) - merged_model.network_meta = attr_to_dict(instance=merged_model.network, sanitize_to_strings=True) - merged_model.included = [model['pmd:TSO'] for model in input_models if model.get('pmd:TSO', None)] - - # Crosscheck replaced model outages with latest UAP if at least one Baltic model was replaced - replaced_tso_list = [entity['tso'] for entity in merged_model.replaced_entity] - - # Update model outages - tso_list = [] - if force_outage_fix: # force outage fix on all models if set - tso_list = merged_model.included - elif outage_update and merging_area == 'BA' and any(tso in ['LITGRID', 'AST', 'ELERING'] for tso in - replaced_tso_list): # by default do it on Baltic merge replaced models - tso_list = replaced_tso_list - if tso_list: # if not set force and not replaced BA then nothing to fix - merged_model = merge_functions.update_model_outages(merged_model=merged_model, - tso_list=tso_list, - scenario_datetime=scenario_datetime, - time_horizon=time_horizon) - - # Various corrections from igmsshvscgmssh error - if json.loads(REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION.lower()): - merged_model.network = handle_igm_ssh_vs_cgm_ssh_error(network_pre_instance=merged_model.network) - - # Ensure boundary point EquivalentInjection are set to zero for paired tie lines - merged_model.network = merge_functions.ensure_paired_equivalent_injection_compatibility( - network=merged_model.network) - - # Ensure boundary line connectivity consistency for paired boundary lines - merged_model.network = merge_functions.ensure_paired_boundary_line_connectivity(network=merged_model.network) - - # Run loadflow on merged model - merged_model, pp_loadflow_parameters = self.run_loadflow(merged_model=merged_model) - logger.info( - f"Loadflow status of main island: {merged_model.loadflow_status} [settings: {merged_model.loadflow_settings}]") - - # Perform scaling - if model_scaling: - - # Scale balance if all schedules were received - if all([ac_schedules, dc_schedules]): - try: - merged_model = scaler.scale_balance(model=merged_model, - ac_schedules=ac_schedules, - dc_schedules=dc_schedules, - lf_settings=pp_loadflow_parameters) - except Exception as e: - logger.error(e) - merged_model.scaled = False - else: - logger.warning(f"Schedule reference data not available: {schedule_time_horizon} for {schedule_start}") - logger.warning(f"Network model schedule scaling not performed") - merged_model.scaled = False - - # Record main merging process end - merge_end = datetime.datetime.now(datetime.UTC) - - # Update time_horizon in case of generic ID process type - if time_horizon.upper() == "ID": - time_horizon = merge_functions.set_intraday_time_horizon(scenario_datetime, task_creation_time) - logger.info(f"Setting intraday time horizon to: {time_horizon}") - merged_model.time_horizon_id = time_horizon - - # Set merged model name - model_type = "RMM" if merging_area == "BA" else "CGM" - merged_model.name = f"{model_type}_{time_horizon}_{version}_{parse_datetime(scenario_datetime):%Y%m%dT%H%MZ}_{merging_area}_{uuid4()}" - - # Crate OPDM object for merged model - opdm_object_meta = merge_functions.create_merged_model_opdm_object( - object_id=merged_model.network_meta['id'].split("uuid:")[-1], - time_horizon=time_horizon, - merging_entity=merging_entity, - merging_area=merging_area, - scenario_date=scenario_datetime, - mas=mas, - version=version, - ) - # Export merged model - # TODO change here to export SSH profiles as well - exported_model = merge_functions.export_merged_model(network=merged_model.network, - opdm_object_meta=opdm_object_meta, - profiles=["SV"], - cgm_convention=False) - - # Run post-processing - post_p_start = datetime.datetime.now(datetime.UTC) - logger.info(f"Starting merged model post-processing") - # TODO here should be one existing network structure. IIDM model can be exported and removed to release memory - sv_data, ssh_data, opdm_object_meta = merge_functions.run_post_merge_processing(input_models=input_models, - exported_model=exported_model, - opdm_object_meta=opdm_object_meta, - enable_temp_fixes=post_temp_fixes, - task_properties=task_properties - ) - - # for merge report need to get the final uuid. - merged_model.network_meta['fullModel_ID'] = opdm_object_meta['pmd:fullModel_ID'] - # Package both input models and exported CGM profiles to in memory zip files - serialized_data = export_to_cgmes_zip([ssh_data, sv_data]) - post_p_end = datetime.datetime.now(datetime.UTC) - logger.debug(f"Post processing took: {(post_p_end - post_p_start).total_seconds()} seconds") - logger.debug(f"Merging took: {(merge_end - merge_start).total_seconds()} seconds") - - # Upload to OPDM - if model_upload_to_opdm: - if merged_model.loadflow[0][ - 'status'] == 'CONVERGED' and merged_model.scaled: # Only upload if the model LF is solved and scaled = true - try: - self.opdm_service = opdm.OPDM() - - if SEND_TYPE == 'SOAP': - for item in serialized_data: - logger.info(f"Uploading to OPDM: {item.name}") - # time.sleep(1) tested before with 2 and 4. Currently removing as FS gonna be default. - async_call(function=self.opdm_service.publication_request, - callback=log_opdm_response, - file_path_or_file_object=item) - elif SEND_TYPE == 'FS': - for item in serialized_data: - logger.info(f"Uploading to OPDM: {item.name}") - self.opdm_service.put_file(file_id=item.name, - file_content=item) - - merged_model.uploaded_to_opde = True - except Exception as error: - logging.error(f"Unexpected error on uploading to OPDM: {error}", exc_info=True) - else: - logger.info(f"Model not uploaded to OPDM due to convergence or failed scaling issues") - - # Create zipped model data - merged_model_object = BytesIO() - with ZipFile(merged_model_object, "w") as merged_model_zip: - # Include CGM model files - for item in serialized_data: - merged_model_zip.writestr(item.name, item.getvalue()) - - # Include original IGM files - for input_model in input_models: - for instance in input_model['opde:Component']: - if instance['opdm:Profile']['pmd:cgmesProfile'] in ['EQ', 'TP', 'EQBD', 'TPBD', 'EQ_BD', 'TP_BD']: - file_object = get_opdm_component_data_bytes(opdm_component=instance) - logging.info(f"Adding file: {file_object.name}") - merged_model_zip.writestr(file_object.name, file_object.getvalue()) - saved_horizon = str(task_properties["time_horizon"]).strip().upper() - folder = f"{OUTPUT_MINIO_FOLDER}/{saved_horizon}" - merged_model_object.name = f"{folder}/{merged_model.name}.zip" - - # Upload to Minio storage - if model_upload_to_minio: - logger.info(f"Uploading merged model to MINIO: {merged_model_object.name}") - minio_metadata = merge_functions.evaluate_trustability(merged_model.__dict__, task['task_properties']) - try: - response = self.minio_service.upload_object(file_path_or_file_object=merged_model_object, - bucket_name=OUTPUT_MINIO_BUCKET, - metadata=minio_metadata, - ) - if response: - merged_model.uploaded_to_minio = True - - # Optionally upload a second copy only when OUTPUT_MINIO_COPY_FOLDER is set (non-empty). - copy_folder_base = str(globals().get('OUTPUT_MINIO_COPY_FOLDER', '') or '').strip() - if copy_folder_base: - original_name = merged_model_object.name - try: - copy_folder = f"{copy_folder_base}/{saved_horizon}" - copy_name = f"{copy_folder}/{merged_model.name}.zip" - merged_model_object.name = copy_name - merged_model_object.seek(0) - - logger.info( - f"Uploading secondary copy of merged model to MINIO: {merged_model_object.name} " - f"(bucket: {OUTPUT_MINIO_BUCKET})" - ) - self.minio_service.upload_object( - file_path_or_file_object=merged_model_object, - bucket_name=OUTPUT_MINIO_BUCKET, - metadata=minio_metadata, - ) - except Exception as copy_error: - logger.error(f"Failed to upload secondary copy to Minio: {copy_error}", exc_info=True) - finally: - # Restore original name so content_reference remains pointing to original location - merged_model_object.name = original_name - - except Exception as error: - logging.error(f"Unexpected error on uploading to Object Storage: {error}", exc_info=True) - logger.info(f"Merged model creation done for: {merged_model.name}") - - end_time = datetime.datetime.now(datetime.UTC) - task_duration = end_time - start_time - logger.info(f"Task ended at {end_time}, total run time {task_duration}", - extra={"task_duration": task_duration.total_seconds(), - "task_start_time": start_time.isoformat(), - "task_end_time": end_time.isoformat()}) - - # Set task to finished - update_task_status(task, "finished") - logger.debug(task) - - # Update merged model attributes - merged_model.duration_s = (end_time - merge_start).total_seconds() - merged_model.content_reference = merged_model_object.name - - # Update OPDM object data with CGM relevant data and send to Elastic - opdm_object_meta['pmd:content-reference'] = merged_model.content_reference - response = elastic.Elastic.send_to_elastic(index=OPDE_MODELS_ELK_INDEX, json_message=opdm_object_meta) - - # Send merge report and OPDM object metadata to Elastic - merge_report = None - if model_merge_report_send_to_elk: - logger.info(f"Sending merge report to Elastic") - try: - merge_report = merge_functions.generate_merge_report(merged_model=merged_model, task=task) - try: - response = elastic.Elastic.send_to_elastic(index=MERGE_REPORT_ELK_INDEX, json_message=merge_report) - except Exception as error: - logger.error(f"Merge report sending to Elastic failed: {error}") - except Exception as error: - logger.error(f"Failed to create merge report: {error}") - - # Send QAS level 8 report if configured - if lvl8_reporting and merge_report: - try: - lvl8_report = merge_functions.lvl8_report_cgm(merge_report=merge_report) - service_edx = edx.EDX() - message_id = service_edx.send_message(receiver_EIC=QAS_EIC, - business_type=QAS_MSG_TYPE, - content=lvl8_report) - logger.info(f"QAS-Level-8 report generated and sent with ID: {message_id}") - except Exception as error: - logger.error(f"Failed to send QAS-Level-8 report with error: {error}") - else: - logger.warning(f"QAS-Level-8 not generated because merge report unavailable or disabled by configuration") - - # Append message headers with OPDM root metadata - extracted_meta = {key: value for key, value in opdm_object_meta.items() if isinstance(value, str)} - properties.headers = extracted_meta - - # Stop Trace - self.elk_logging_handler.stop_trace() - - logger.info(f"Merge task finished for model: {merged_model.name}") - - return json.dumps(opdm_object_meta), properties - - -if __name__ == "__main__": - sample_task = { - "@context": "https://example.com/task_context.jsonld", - "@type": "Task", - "@id": "urn:uuid:ee3c57bf-fa4e-402c-82ac-7352c0d8e118", - "process_id": "https://example.com/processes/CGM_CREATION", - "run_id": "https://example.com/runs/IntraDayCGM/1", - "job_id": "urn:uuid:d9343f48-23cd-4d8a-ae69-1940a0ab1837", - "task_type": "automatic", - "task_initiator": "cgm_handler_unit_test", - "task_priority": "normal", - "task_creation_time": "2024-11-28T20:39:42.448064", - "task_update_time": "", - "task_status": "created", - "task_status_trace": [ - { - "status": "created", - "timestamp": "2024-11-28T20:39:42.448064" - } - ], - "task_dependencies": [], - "task_tags": [], - "task_retry_count": 0, - "task_timeout": "PT1H", - "task_gate_open": "2024-05-24T21:00:00+00:00", - "task_gate_close": "2024-05-24T21:15:00+00:00", - "job_period_start": "2024-05-24T22:00:00+00:00", - "job_period_end": "2024-05-25T06:00:00+00:00", - "task_properties": { - "timestamp_utc": "2026-07-13T15:30:00+00:00", - "merge_type": "BA", - "merging_entity": "BALTICRCC", - "included": ["PSE", "LITGRID", "ELERING", "AST"], - "excluded": [], - "local_import": [], - "replace_tso": [], - "time_horizon": "1D", - "version": "000", - "mas": "http://www.baltic-rsc.eu/OperationalPlanning", - "post_temp_fixes": "True", - "replacement": "True", - "scaling": "True", - "outage_update": "True", - "force_outage_fix": "False", - "upload_to_opdm": "False", - "upload_to_minio": "True", - "send_merge_report": "True", - "lvl8_reporting": "False" - } - } - class Properties(dict): - pass - properties = Properties() - properties.headers = {} - - worker = HandlerMergeModels() - finished_task = worker.handle(sample_task, {}) \ No newline at end of file +import logging +import pypowsybl +import config +import json +import time +from uuid import uuid4 +import datetime +from dataclasses import dataclass, field +from typing import List +from emf.common.helpers.time import parse_datetime +from io import BytesIO +from zipfile import ZipFile +from emf.common.config_parser import parse_app_properties +from emf.common.integrations import opdm, minio_api, elastic, edx +from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download +from emf.common.integrations.object_storage.schedules import query_acnp_schedules, query_hvdc_schedules, \ + calculate_ac_net_position +from emf.common.loadflow_tool import loadflow_settings, settings_manager +from emf.common.helpers.utils import attr_to_dict, convert_dict_str_to_bool +from emf.common.helpers.cgmes import export_to_cgmes_zip +from emf.common.helpers.opdm_objects import get_opdm_component_data_bytes +from emf.common.helpers.loadflow import load_network_model +from emf.common.helpers.tasks import update_task_status +from emf.model_merger import merge_functions +from emf.model_merger import scaler +from emf.model_merger.merge_functions import filter_models_by_acnp +from emf.model_merger.replacement import run_replacement, get_tsos_available_in_storage +from emf.model_merger.temporary import handle_igm_ssh_vs_cgm_ssh_error +from emf.common.logging.custom_logger import get_elk_logging_handler +from concurrent.futures import ThreadPoolExecutor +from lxml import etree + +logger = logging.getLogger(__name__) +parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.merger) +executor = ThreadPoolExecutor(max_workers=20) + +# Retrieve the list of all TSOs ever +config_areas_mapping = config.paths.cgm_worker.config_areas_mapping +tsos_config_json = json.load(config_areas_mapping) +possible_tsos = [area['party.name'] for area in tsos_config_json if 'party.name' in area] + +def async_call(function, callback=None, *args, **kwargs): + future = executor.submit(function, *args, **kwargs) + if callback: + future.add_done_callback(lambda f: callback(f.result())) + return future + + +def log_opdm_response(response): + logger.debug(etree.tostring(response, pretty_print=True).decode()) + + +@dataclass +class MergedModel: + network: pypowsybl.network = None + time_horizon: str = None + time_horizon_id: str = field(default_factory=str) + name = None + loadflow_status: str | None = None + + # Status flags + scaled: bool = None + replaced: bool = None + outages: bool = None + uploaded_to_opde: bool = False + uploaded_to_minio: bool = False + + # Extended data + loadflow: List = field(default_factory=list) + excluded: List = field(default_factory=list) + scaled_entity: List = field(default_factory=list) + scaled_hvdc: List = field(default_factory=list) + replaced_entity: List = field(default_factory=list) + replacement_reason: List = field(default_factory=list) + outages_updated: List = field(default_factory=list) + outages_unmapped: List = field(default_factory=list) + merge_included_entity: List = field(default_factory=list) + + +@dataclass(init=False) +class ModelEntity: + data_source: str = "OPDM" + quality_indicator: str = "Valid" + tso: str = None + time_horizon: str = None + scenario_timestamp: str = None + model_sv_id: str = None + version: int = None + quality_indicator: str = "Valid" + creation_timestamp: str = None + file_name: str = None + + def __init__(self, data_source: str, quality_indicator: str, **kwargs): + self.data_source = data_source + self.quality_indicator = quality_indicator + self.tso = kwargs.get('pmd:TSO', 'unknown') + self.time_horizon = kwargs.get('pmd:timeHorizon', 'unknown') + self.scenario_timestamp = kwargs.get('pmd:scenarioDate', 'unknown') + self.model_sv_id = kwargs.get('pmd:fullModel_ID', 'unknown') + self.version = int(kwargs.get('pmd:version', 999)) + self.creation_timestamp = kwargs.get('pmd:creationDate', 'unknown') + self.file_name = kwargs.get('pmd:fileName', 'unknown') + + +class HandlerMergeModels: + + def __init__(self): + self.minio_service = minio_api.ObjectStorage() + self.elk_logging_handler = get_elk_logging_handler() + self.opdm_service = None + + @staticmethod + def run_loadflow(merged_model): + # Set starting point of lf settings priority list + if json.loads(ENABLE_DYNAMIC_MERGE_SETTINGS.lower()): + settings_list = [param.strip() for param in MERGE_LOAD_FLOW_SETTINGS_PRIORITY.split(",")] + settings_priority = next((i for i, value in enumerate(settings_list) if value == MERGE_LOAD_FLOW_SETTINGS), + None) + settings_list = settings_list[settings_priority:] + else: + settings_list = [MERGE_LOAD_FLOW_SETTINGS] + + for lf_settings in settings_list: + logger.info(f"Solving loadflow with settings: {lf_settings}") + # report = pypowsybl.report.Reporter() + manager = settings_manager.LoadflowSettingsManager(settings_keyword=lf_settings) + pp_loadflow_parameters = manager.build_pypowsybl_parameters() + result = pypowsybl.loadflow.run_ac(network=merged_model.network, + parameters=pp_loadflow_parameters, + # reporter=loadflow_report, + ) + if result[0].status_text == 'Converged': + break + else: + logger.warning(f"Failed to solve loadflow with settings: {lf_settings}") + + result_dict = [attr_to_dict(island) for island in result] + # Modify all nested objects to native data types + for island in result_dict: + island['status'] = island['status'].name + # Extract only first slack bus from internal pypowsybl object + slack_bus_results = island.pop('slack_bus_results') + if slack_bus_results: + island['slack_bus_id'] = getattr(slack_bus_results[0], 'id', 'undefined') + island['active_power_mismatch'] = getattr(slack_bus_results[0], 'active_power_mismatch', float()) + else: + island['slack_bus_id'] = 'undefined' + island['active_power_mismatch'] = float() + + # merged_model.loadflow = json.loads(loadflow_report.to_json()) + # merged_model.loadflow = str(loadflow_report) + merged_model.loadflow = [island for island in result_dict if island['reference_bus_id']] + merged_model.loadflow_status = result[0].status.name # store main island loadflow status + merged_model.loadflow_settings = lf_settings + + return merged_model, pp_loadflow_parameters + + def handle(self, task_object: dict, properties: dict, **kwargs): + + start_time = datetime.datetime.now(datetime.UTC) + + # Create instance of merged model + merged_model = MergedModel() + + # Parse relevant data from Task + task = task_object + if not isinstance(task, dict): + task = json.loads(task_object) + + # Convert task fields to bool where necessary + task = convert_dict_str_to_bool(task) + + # TODO - make it to a wrapper once it is settled/standardized how this info is exchanged + # Initialize trace + self.elk_logging_handler.start_trace(task) + logger.debug(task) + + # Set task to started + update_task_status(task, "started") + + # Task configuration + task_creation_time = task.get('task_creation_time') + task_properties = task.get('task_properties', {}) + included_models = task_properties.get('included', []) + excluded_models = task_properties.get('excluded', []) + local_import_models = task_properties.get('local_import', []) + replace_tso = task_properties.get('replace_tso', []) + time_horizon = task_properties["time_horizon"] + scenario_datetime = task_properties["timestamp_utc"] + schedule_start = task_properties.get("reference_schedule_start_utc") + schedule_end = task_properties.get("reference_schedule_end_utc") + schedule_time_horizon = task_properties.get("reference_schedule_time_horizon") + merging_area = task_properties["merge_type"] + merging_entity = task_properties["merging_entity"] + mas = task_properties["mas"] + version = task_properties["version"] + model_replacement = task_properties["replacement"] + model_scaling = task_properties["scaling"] + outage_update = task_properties["outage_update"] + force_outage_fix = task_properties['force_outage_fix'] + model_upload_to_opdm = task_properties["upload_to_opdm"] + model_upload_to_minio = task_properties["upload_to_minio"] + model_merge_report_send_to_elk = task_properties["send_merge_report"] + post_temp_fixes = task_properties['post_temp_fixes'] + lvl8_reporting = task_properties['lvl8_reporting'] + + # Get aligned schedules + # Set default time horizon and scenario timestamp if not provided + if not schedule_time_horizon or schedule_time_horizon == "AUTO": + schedule_time_horizon = time_horizon + + if not schedule_start: + schedule_start = scenario_datetime + + ac_schedules = query_acnp_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) + dc_schedules = query_hvdc_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) + acnp_dict = calculate_ac_net_position(ac_schedules) + + + # Create list of only the TSOs that are needed for the merge, to query Elastic only for *their* metadata + desired_tsos = merge_functions.filter_tsos(tsos=possible_tsos, + included_models=included_models, + excluded_models=excluded_models) + logger.info(f"Compiled the desired tsos list as {desired_tsos}") + # but since that did not take into account local import desired TSOs, have to add those as well: + if local_import_models: desired_tsos = desired_tsos+local_import_models + + # Collect valid models from ObjectStorage (this is just the metadata of the models, not the actual zips) + models = get_latest_models_and_download(time_horizon=time_horizon, + scenario_date=scenario_datetime, + valid=True, + tso=desired_tsos, + data_source='OPDM') + latest_boundary = get_latest_boundary() + + # Get additional models from ObjectStorage if local import is configured + if local_import_models: + additional_models = get_latest_models_and_download(time_horizon=time_horizon, + scenario_date=scenario_datetime, + valid=True, + tso=desired_tsos, + data_source='PDN') + + additional_tsos = {model['pmd:TSO'] for model in additional_models} + missing_local_import = [tso for tso in local_import_models if tso not in additional_tsos] + merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-pdn'} for tso in missing_local_import]) + + else: + additional_models = [] + missing_local_import = [] + + # Check missing models for replacement + if included_models: + models_tsos = {model['pmd:TSO'] for model in models} + missing_models = [tso for tso in included_models if tso not in models_tsos] + + if missing_models: + merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-opdm'} for tso in missing_models]) + + # find RMM models: + missing_models_rmm = [tso for tso in missing_models if merging_area == "BA"] + + if missing_models_rmm: + # Get PDN models when OPDM models missing + pdn_auto_models = get_latest_models_and_download(time_horizon=time_horizon, + scenario_date=scenario_datetime, + valid=True, + tso=desired_tsos, + data_source='PDN') + + # Cache PDN TSO set + pdn_tsos = {m['pmd:TSO'] for m in pdn_auto_models} + missing_pdn_auto = [tso for tso in missing_models_rmm if tso not in pdn_tsos] + + if missing_pdn_auto: + logging.info(f"OPDM and PDN missing for {missing_pdn_auto}") + + replaced_with_pdn = [tso for tso in missing_models_rmm if tso not in missing_pdn_auto] + if replaced_with_pdn: + logging.info(f"OPDM missing for {replaced_with_pdn} - replaced with PDN models") + + models = models + pdn_auto_models + + # Update exclusion list reason + for item in merged_model.excluded: + if item['tso'] in missing_pdn_auto: + item['reason'] = 'missing-opdm-and-pdn' + + # Rewrite missing_models + missing_models = [tso for tso in missing_models if tso not in pdn_tsos] + + else: + if model_replacement: + # Get TSOs which models are available in storage for replacement period + available_tsos = get_tsos_available_in_storage(time_horizon=time_horizon) + valid_model_tsos = [model['pmd:TSO'] for model in models] + # Need to ensure that excluded models by task configuration would not be taken in replacement context + missing_models = [tso for tso in available_tsos if tso not in valid_model_tsos + excluded_models] + if missing_models: + merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-opdm'} for tso in missing_models]) + else: + missing_models = [] + + # Exclude models that are outside scheduled AC net position deadband + if acnp_dict: + logger.info("Excluding models with incorrect ACNP") + filterd_models = filter_models_by_acnp(models, merged_model, acnp_dict, ACNP_THRESHOLD, CONFORM_LOAD_FACTOR) + filterd_additional_models = filter_models_by_acnp(additional_models, merged_model, acnp_dict, ACNP_THRESHOLD, + CONFORM_LOAD_FACTOR) + if included_models: + # Update missing models + missing_models = [m for m in included_models if m not in [m['pmd:TSO'] for m in filterd_models]] + missing_local_import = [m for m in local_import_models if m not in [m['pmd:TSO'] for m in filterd_additional_models]] + models = filterd_models + additional_models = filterd_additional_models + elif model_replacement: + models_tsos = {model['pmd:TSO'] for model in models} + excluded_incorrect = [tso for tso in valid_model_tsos if + tso not in models_tsos and tso not in missing_models] + missing_models = missing_models + excluded_incorrect + + # Execute consolidated model replacement logic + models, additional_models = run_replacement( + models=models, + additional_models=additional_models, + model_replacement=model_replacement, + local_import_models=local_import_models, + missing_local_import=missing_local_import, + missing_models=missing_models, + replace_tso=replace_tso, + time_horizon=time_horizon, + scenario_datetime=scenario_datetime, + merged_model=merged_model, + acnp_dict=acnp_dict, + acnp_threshold=ACNP_THRESHOLD, + conform_load_factor=CONFORM_LOAD_FACTOR + ) + + merged_model.merge_included_entity = [ + ModelEntity(data_source='OPDM', quality_indicator='Valid', **model).__dict__ for model in models] + + merged_model.merge_included_entity.extend( + [ModelEntity(data_source='PDN', quality_indicator='Valid', **model).__dict__ for model in + additional_models]) + + # Store models together with boundary set and check whether there are enough models to merge + input_models = models + additional_models + [latest_boundary] + if len(input_models) < 2: + logger.warning("No valid models found for merging, exiting merge process") + properties.headers['success'] = False + return task_object, properties + + # Load network model and merge + merge_start = datetime.datetime.now(datetime.UTC) + merged_model.network = load_network_model(opdm_objects=input_models) + merged_model.network_meta = attr_to_dict(instance=merged_model.network, sanitize_to_strings=True) + merged_model.included = [model['pmd:TSO'] for model in input_models if model.get('pmd:TSO', None)] + + # Crosscheck replaced model outages with latest UAP if at least one Baltic model was replaced + replaced_tso_list = [entity['tso'] for entity in merged_model.replaced_entity] + + # Update model outages + tso_list = [] + if force_outage_fix: # force outage fix on all models if set + tso_list = merged_model.included + elif outage_update and merging_area == 'BA' and any(tso in ['LITGRID', 'AST', 'ELERING'] for tso in + replaced_tso_list): # by default do it on Baltic merge replaced models + tso_list = replaced_tso_list + if tso_list: # if not set force and not replaced BA then nothing to fix + merged_model = merge_functions.update_model_outages(merged_model=merged_model, + tso_list=tso_list, + scenario_datetime=scenario_datetime, + time_horizon=time_horizon) + + # Various corrections from igmsshvscgmssh error + if json.loads(REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION.lower()): + merged_model.network = handle_igm_ssh_vs_cgm_ssh_error(network_pre_instance=merged_model.network) + + # Ensure boundary point EquivalentInjection are set to zero for paired tie lines + merged_model.network = merge_functions.ensure_paired_equivalent_injection_compatibility( + network=merged_model.network) + + # Ensure boundary line connectivity consistency for paired boundary lines + merged_model.network = merge_functions.ensure_paired_boundary_line_connectivity(network=merged_model.network) + + # Run loadflow on merged model + merged_model, pp_loadflow_parameters = self.run_loadflow(merged_model=merged_model) + logger.info( + f"Loadflow status of main island: {merged_model.loadflow_status} [settings: {merged_model.loadflow_settings}]") + + # Perform scaling + if model_scaling: + + # Scale balance if all schedules were received + if all([ac_schedules, dc_schedules]): + try: + merged_model = scaler.scale_balance(model=merged_model, + ac_schedules=ac_schedules, + dc_schedules=dc_schedules, + lf_settings=pp_loadflow_parameters) + except Exception as e: + logger.error(e) + merged_model.scaled = False + else: + logger.warning(f"Schedule reference data not available: {schedule_time_horizon} for {schedule_start}") + logger.warning(f"Network model schedule scaling not performed") + merged_model.scaled = False + + # Record main merging process end + merge_end = datetime.datetime.now(datetime.UTC) + + # Update time_horizon in case of generic ID process type + if time_horizon.upper() == "ID": + time_horizon = merge_functions.set_intraday_time_horizon(scenario_datetime, task_creation_time) + logger.info(f"Setting intraday time horizon to: {time_horizon}") + merged_model.time_horizon_id = time_horizon + + # Set merged model name + model_type = "RMM" if merging_area == "BA" else "CGM" + merged_model.name = f"{model_type}_{time_horizon}_{version}_{parse_datetime(scenario_datetime):%Y%m%dT%H%MZ}_{merging_area}_{uuid4()}" + + # Crate OPDM object for merged model + opdm_object_meta = merge_functions.create_merged_model_opdm_object( + object_id=merged_model.network_meta['id'].split("uuid:")[-1], + time_horizon=time_horizon, + merging_entity=merging_entity, + merging_area=merging_area, + scenario_date=scenario_datetime, + mas=mas, + version=version, + ) + # Export merged model + # TODO change here to export SSH profiles as well + exported_model = merge_functions.export_merged_model(network=merged_model.network, + opdm_object_meta=opdm_object_meta, + profiles=["SV"], + cgm_convention=False) + + # Run post-processing + post_p_start = datetime.datetime.now(datetime.UTC) + logger.info(f"Starting merged model post-processing") + # TODO here should be one existing network structure. IIDM model can be exported and removed to release memory + sv_data, ssh_data, opdm_object_meta = merge_functions.run_post_merge_processing(input_models=input_models, + exported_model=exported_model, + opdm_object_meta=opdm_object_meta, + enable_temp_fixes=post_temp_fixes, + task_properties=task_properties + ) + + # for merge report need to get the final uuid. + merged_model.network_meta['fullModel_ID'] = opdm_object_meta['pmd:fullModel_ID'] + # Package both input models and exported CGM profiles to in memory zip files + serialized_data = export_to_cgmes_zip([ssh_data, sv_data]) + post_p_end = datetime.datetime.now(datetime.UTC) + logger.debug(f"Post processing took: {(post_p_end - post_p_start).total_seconds()} seconds") + logger.debug(f"Merging took: {(merge_end - merge_start).total_seconds()} seconds") + + # Upload to OPDM + if model_upload_to_opdm: + if merged_model.loadflow[0][ + 'status'] == 'CONVERGED' and merged_model.scaled: # Only upload if the model LF is solved and scaled = true + try: + self.opdm_service = opdm.OPDM() + + if SEND_TYPE == 'SOAP': + for item in serialized_data: + logger.info(f"Uploading to OPDM: {item.name}") + # time.sleep(1) tested before with 2 and 4. Currently removing as FS gonna be default. + async_call(function=self.opdm_service.publication_request, + callback=log_opdm_response, + file_path_or_file_object=item) + elif SEND_TYPE == 'FS': + for item in serialized_data: + logger.info(f"Uploading to OPDM: {item.name}") + self.opdm_service.put_file(file_id=item.name, + file_content=item) + + merged_model.uploaded_to_opde = True + except Exception as error: + logging.error(f"Unexpected error on uploading to OPDM: {error}", exc_info=True) + else: + logger.info(f"Model not uploaded to OPDM due to convergence or failed scaling issues") + + # Create zipped model data + merged_model_object = BytesIO() + with ZipFile(merged_model_object, "w") as merged_model_zip: + # Include CGM model files + for item in serialized_data: + merged_model_zip.writestr(item.name, item.getvalue()) + + # Include original IGM files + for input_model in input_models: + for instance in input_model['opde:Component']: + if instance['opdm:Profile']['pmd:cgmesProfile'] in ['EQ', 'TP', 'EQBD', 'TPBD', 'EQ_BD', 'TP_BD']: + file_object = get_opdm_component_data_bytes(opdm_component=instance) + logging.info(f"Adding file: {file_object.name}") + merged_model_zip.writestr(file_object.name, file_object.getvalue()) + saved_horizon = str(task_properties["time_horizon"]).strip().upper() + folder = f"{OUTPUT_MINIO_FOLDER}/{saved_horizon}" + merged_model_object.name = f"{folder}/{merged_model.name}.zip" + + # Upload to Minio storage + if model_upload_to_minio: + logger.info(f"Uploading merged model to MINIO: {merged_model_object.name}") + minio_metadata = merge_functions.evaluate_trustability(merged_model.__dict__, task['task_properties']) + try: + response = self.minio_service.upload_object(file_path_or_file_object=merged_model_object, + bucket_name=OUTPUT_MINIO_BUCKET, + metadata=minio_metadata, + ) + if response: + merged_model.uploaded_to_minio = True + + # Optionally upload a second copy only when OUTPUT_MINIO_COPY_FOLDER is set (non-empty). + copy_folder_base = str(globals().get('OUTPUT_MINIO_COPY_FOLDER', '') or '').strip() + if copy_folder_base: + original_name = merged_model_object.name + try: + copy_folder = f"{copy_folder_base}/{saved_horizon}" + copy_name = f"{copy_folder}/{merged_model.name}.zip" + merged_model_object.name = copy_name + merged_model_object.seek(0) + + logger.info( + f"Uploading secondary copy of merged model to MINIO: {merged_model_object.name} " + f"(bucket: {OUTPUT_MINIO_BUCKET})" + ) + self.minio_service.upload_object( + file_path_or_file_object=merged_model_object, + bucket_name=OUTPUT_MINIO_BUCKET, + metadata=minio_metadata, + ) + except Exception as copy_error: + logger.error(f"Failed to upload secondary copy to Minio: {copy_error}", exc_info=True) + finally: + # Restore original name so content_reference remains pointing to original location + merged_model_object.name = original_name + + except Exception as error: + logging.error(f"Unexpected error on uploading to Object Storage: {error}", exc_info=True) + logger.info(f"Merged model creation done for: {merged_model.name}") + + end_time = datetime.datetime.now(datetime.UTC) + task_duration = end_time - start_time + logger.info(f"Task ended at {end_time}, total run time {task_duration}", + extra={"task_duration": task_duration.total_seconds(), + "task_start_time": start_time.isoformat(), + "task_end_time": end_time.isoformat()}) + + # Set task to finished + update_task_status(task, "finished") + logger.debug(task) + + # Update merged model attributes + merged_model.duration_s = (end_time - merge_start).total_seconds() + merged_model.content_reference = merged_model_object.name + + # Update OPDM object data with CGM relevant data and send to Elastic + opdm_object_meta['pmd:content-reference'] = merged_model.content_reference + response = elastic.Elastic.send_to_elastic(index=OPDE_MODELS_ELK_INDEX, json_message=opdm_object_meta) + + # Send merge report and OPDM object metadata to Elastic + merge_report = None + if model_merge_report_send_to_elk: + logger.info(f"Sending merge report to Elastic") + try: + merge_report = merge_functions.generate_merge_report(merged_model=merged_model, task=task) + try: + response = elastic.Elastic.send_to_elastic(index=MERGE_REPORT_ELK_INDEX, json_message=merge_report) + except Exception as error: + logger.error(f"Merge report sending to Elastic failed: {error}") + except Exception as error: + logger.error(f"Failed to create merge report: {error}") + + # Send QAS level 8 report if configured + if lvl8_reporting and merge_report: + try: + lvl8_report = merge_functions.lvl8_report_cgm(merge_report=merge_report) + service_edx = edx.EDX() + message_id = service_edx.send_message(receiver_EIC=QAS_EIC, + business_type=QAS_MSG_TYPE, + content=lvl8_report) + logger.info(f"QAS-Level-8 report generated and sent with ID: {message_id}") + except Exception as error: + logger.error(f"Failed to send QAS-Level-8 report with error: {error}") + else: + logger.warning(f"QAS-Level-8 not generated because merge report unavailable or disabled by configuration") + + # Append message headers with OPDM root metadata + extracted_meta = {key: value for key, value in opdm_object_meta.items() if isinstance(value, str)} + properties.headers = extracted_meta + + # Stop Trace + self.elk_logging_handler.stop_trace() + + logger.info(f"Merge task finished for model: {merged_model.name}") + + return json.dumps(opdm_object_meta), properties + + +if __name__ == "__main__": + sample_task = { + "@context": "https://example.com/task_context.jsonld", + "@type": "Task", + "@id": "urn:uuid:ee3c57bf-fa4e-402c-82ac-7352c0d8e118", + "process_id": "https://example.com/processes/CGM_CREATION", + "run_id": "https://example.com/runs/IntraDayCGM/1", + "job_id": "urn:uuid:d9343f48-23cd-4d8a-ae69-1940a0ab1837", + "task_type": "automatic", + "task_initiator": "cgm_handler_unit_test", + "task_priority": "normal", + "task_creation_time": "2024-11-28T20:39:42.448064", + "task_update_time": "", + "task_status": "created", + "task_status_trace": [ + { + "status": "created", + "timestamp": "2024-11-28T20:39:42.448064" + } + ], + "task_dependencies": [], + "task_tags": [], + "task_retry_count": 0, + "task_timeout": "PT1H", + "task_gate_open": "2024-05-24T21:00:00+00:00", + "task_gate_close": "2024-05-24T21:15:00+00:00", + "job_period_start": "2024-05-24T22:00:00+00:00", + "job_period_end": "2024-05-25T06:00:00+00:00", + "task_properties": { + "timestamp_utc": "2026-07-17T10:30:00+00:00", # replacement testing + #"timestamp_utc": "2026-07-16T17:30:00+00:00", # no replacements + #"merge_type": "EU", + "merge_type": "BA", + "merging_entity": "BALTICRCC", + #"included": ["PSE", "LITGRID", "AST"], + "included": ["PSE"], + #"excluded": ["DKE", "DKW", "D7", "UKRENERGO", "TEIAS"], + # EXCLUDING ALL BUT ELIA, TTG, 50Hertz FOR TESTING + #"excluded": ["ELERING", "EIRGRID", "TRANSELECTRICA", "UA_WPS", "KOSOVA-TSMO", "OST", "APG", "NOSBIH", "ESO", "SWISSGRID", "CGES", "EMS", "CEPS", "D4", "REE", "FI", "RTEFRANCE", "NGET", "IPTO", "HOPS", "MAVIR", "TERNA", "LITGRID", "CREOS", "AST", "REE", "MEPSO", "TTN", "STATNETT", "PSE", "REN", "Transelectrica", "SVK", "ELES", "SEPS", "TEIAS", "UKRENERGO"], + "local_import": ["ELERING", "AST", "LITGRID"], + "replace_tso": [], + "time_horizon": "1D", + "version": "000", + "mas": "http://www.baltic-rsc.eu/OperationalPlanning", + "post_temp_fixes": "True", + "replacement": "True", + "scaling": "True", + "outage_update": "True", + "force_outage_fix": "False", + "upload_to_opdm": "False", + "upload_to_minio": "False", + "send_merge_report": "False", + "lvl8_reporting": "False" + } + } + + worker = HandlerMergeModels() + finished_task = worker.handle(sample_task, {})