Skip to content

Veiko aunapuu lvl8 #364

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
VeikoAunapuu merged 11 commits into from
Jul 31, 2025
Merged

Veiko aunapuu lvl8 #364

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
78b7868
Func for lvl8
VeikoAunapuu Jul 29, 2025
b22dda4
Integration of lvl 8
VeikoAunapuu Jul 29, 2025
9bfbd58
Update model_merger.py
VeikoAunapuu Jul 29, 2025
b980090
Update merge_functions.py
VeikoAunapuu Jul 29, 2025
1222104
Init_working_version
VeikoAunapuu Jul 30, 2025
94b0925
Added missing dependency
VeikoAunapuu Jul 30, 2025
23ddeb3
fixes, and tested in QAS
VeikoAunapuu Jul 30, 2025
13098d9
Added filename to models for better trace
VeikoAunapuu Jul 30, 2025
c76c65b
temporary commit
m-karo Jul 31, 2025
a693edc
reafactor: changed to use dataclasss for model entity
m-karo Jul 31, 2025
c7b5593
minor fixes
VeikoAunapuu Jul 31, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion config/cgm_worker/merger.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,4 +10,6 @@ OPDE_MODELS_ELK_INDEX = emfos-opde-models
ENABLE_DYNAMIC_MERGE_SETTINGS = True
MERGE_LOAD_FLOW_SETTINGS = CGM_DEFAULT
MERGE_LOAD_FLOW_SETTINGS_PRIORITY = CGM_DEFAULT,CGM_RELAXED_1,CGM_RELAXED_2
REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION = True
REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION = True
QAS_EIC = QAS_EIC
QAS_MSG_TYPE = QAS_MSG_TYPE
56 changes: 42 additions & 14 deletions config/task_generator/process_conf.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,9 @@
"scaling": "False",
"upload_to_opdm": "True",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},
{
Expand Down Expand Up @@ -73,7 +75,9 @@
"scaling": "False",
"upload_to_opdm": "True",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},
{
Expand Down Expand Up @@ -107,7 +111,9 @@
"scaling": "False",
"upload_to_opdm": "True",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},
{
Expand Down Expand Up @@ -141,7 +147,9 @@
"scaling": "False",
"upload_to_opdm": "True",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},
{
Expand Down Expand Up @@ -175,7 +183,9 @@
"scaling": "False",
"upload_to_opdm": "True",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
}
]
Expand Down Expand Up @@ -227,7 +237,9 @@
"scaling": "False",
"upload_to_opdm": "False",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},
{
Expand Down Expand Up @@ -268,7 +280,9 @@
"scaling": "False",
"upload_to_opdm": "False",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},
{
Expand Down Expand Up @@ -309,7 +323,9 @@
"scaling": "False",
"upload_to_opdm": "False",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},
{
Expand Down Expand Up @@ -349,7 +365,9 @@
"scaling": "False",
"upload_to_opdm": "False",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},
{
Expand Down Expand Up @@ -390,7 +408,9 @@
"scaling": "False",
"upload_to_opdm": "False",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},
{
Expand Down Expand Up @@ -431,7 +451,9 @@
"scaling": "False",
"upload_to_opdm": "False",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},
{
Expand Down Expand Up @@ -472,7 +494,9 @@
"scaling": "False",
"upload_to_opdm": "False",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},

Expand Down Expand Up @@ -517,7 +541,9 @@
"scaling": "False",
"upload_to_opdm": "False",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
},

Expand Down Expand Up @@ -562,7 +588,9 @@
"scaling": "False",
"upload_to_opdm": "False",
"upload_to_minio": "True",
"send_merge_report": "True"
"send_merge_report": "True",
"force_outage_fix": "False",
"lvl8_reporting" : "True"
}
}

Expand Down
89 changes: 89 additions & 0 deletions emf/model_merger/merge_functions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,15 @@
import triplets
import uuid
import config
import xml.etree.ElementTree as ET
from emf.common.config_parser import parse_app_properties
from emf.common.integrations import elastic
from emf.model_merger import temporary
from emf.common.helpers.time import parse_datetime
from emf.common.helpers.loadflow import get_model_outages, get_network_elements
from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, filename_from_opdm_metadata


logger = logging.getLogger(__name__)

parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.post_processing)
Expand Down Expand Up @@ -857,6 +859,93 @@ def run_post_merge_processing(input_models: list,
return sv_data, ssh_data, opdm_object_meta


def lvl8_report_cgm(merge_report: dict):

# Create <QAReport> root
qa_attribs = {
'created': datetime.datetime.strptime(merge_report["@timestamp"], '%Y-%m-%dT%H:%M:%S.%f').strftime('%Y-%m-%dT%H:%M:%SZ'),
'schemeVersion': "2.0",
'serviceProvider': merge_report["merge_entity"],
'xmlns': "http://entsoe.eu/checks"
}
qa_root = ET.Element("QAReport", attrib=qa_attribs)

# Add RuleViolations
violations_list = [
{
'ruleId': "CGMConvergence",
'validationLevel': "8",
'severity': "WARNING",
'Message': "Power flow could not be calculated for CGM with default settings."
},
{
'ruleId': "CGMConvergenceRelaxed",
'validationLevel': "8",
'severity': "ERROR",
'Message': "Power flow could not be calculated for CGM with CGM_RELAXED_2 settings."
}
]
# TODO:pick the correct setting based on retruned LF setting and convergance from model. Set model quality indicator based on violations
violations = []
if merge_report["loadflow_status"] == 'CONVERGED':
if merge_report["loadflow_settings"] == 'CGM_DEFAULT':
logger.info(f"Merge successful with default settings included in lvl8 report")
quality_indicator_cgm = "Valid"
else:
violations.append(violations_list[0])
quality_indicator_cgm = "Warning - non fatal inconsistencies"
else:
violations = violations.append(violations_list)
quality_indicator_cgm = "Invalid - inconsistent data"

# Create <CGM>
cgm_attribs = {
'created': datetime.datetime.strptime(merge_report["@timestamp"], '%Y-%m-%dT%H:%M:%S.%f').strftime('%Y-%m-%dT%H:%M:%SZ'),
'resource': merge_report['network_meta']['fullModel_ID'], # TODO get here correct content ID
'scenarioTime': datetime.datetime.fromisoformat(merge_report["@scenario_timestamp"]).strftime('%Y-%m-%dT%H:%M:%SZ'),
'version': str(merge_report["@version"]),
'processType': merge_report["@time_horizon"],
'qualityIndicator': quality_indicator_cgm
}
cgm = ET.SubElement(qa_root, "CGM", attrib=cgm_attribs)

try:
for v in violations:
rv = ET.SubElement(cgm, "RuleViolation", {
'ruleId': v['ruleId'],
'validationLevel': v['validationLevel'],
'severity': v['severity']
})
msg = ET.SubElement(rv, "Message")
msg.text = v['Message']
except:
logger.info(f"No violations present in merge")

# TODO:pick the TSOs from QA report. Missing parameters below for all IGMs
for i in merge_report['merge_included_entity'] + merge_report['replaced_entity']:
igm = ET.SubElement(cgm, "IGM", {
'created': i["creation_timestamp"],
'scenarioTime': datetime.datetime.fromisoformat(i['scenario_timestamp']).strftime('%Y-%m-%dT%H:%M:%SZ'),
'tso': i['tso'],
'version': str(i['version']),
'processType': i['time_horizon'],
'qualityIndicator': i['quality_indicator'],
})
resource_igm = ET.SubElement(igm, "resource")
resource_igm.text = i['model_sv_id']

# Add EMFInformation
ET.SubElement(cgm, "EMFInformation", {
'mergingEntity': merge_report["merge_entity"],
'cgmType': merge_report["merge_type"]
})

# Generate final XML
qa_report_lvl8 = ET.tostring(qa_root, encoding='utf-8', xml_declaration=True)

return qa_report_lvl8


if __name__ == "__main__":

from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download
Expand Down
Loading
Loading