Production release v1.0.20

.github/workflows/publish-workers.yml

@@ -68,3 +68,10 @@ jobs:
           if [[ "$BRANCH_NAME" == *-dev ]]; then
             docker push "${IMAGE_NAME}:latest"
           fi
+  #    - name: Update helm values with new image tag
+  #      run: |
+  #        git config user.name "Github Actions"
+  #        git config user.email "actions@github.com"
+  #        git add helm-chart/test-rcc-cl2.values.yaml
+  #        git commit -m "Update image tag to ${{ env.IMAGE_TAG }}
+  #        git push

.python-version

@@ -0,0 +1 @@
+3.11

Pipfile

@@ -6,7 +6,7 @@ name = "pypi"
 [packages]
 pandas = "2.2.2"
 numpy = "2.0.0"
-pypowsybl = "1.9.0"
+pypowsybl = "*"
 aniso8601 = "*"
 lxml = "*"
 edx = "*"

config/cgm_worker/merger.properties

@@ -1,11 +1,13 @@
 [MAIN]
 INPUT_RABBIT_QUE = emf-merge.tasks.rmm
+OUTPUT_RMQ_EXCHANGE = emf-quality.test
 INPUT_MINIO_BUCKET = opde-confidential-models
 INPUT_MINIO_FOLDER = IGM
 OUTPUT_MINIO_BUCKET = opde-confidential-models
 OUTPUT_MINIO_FOLDER = EMFOS/RMM
 MERGE_REPORT_ELK_INDEX = emfos-merge-reports
-SMALL_ISLAND_SIZE = 10
-MERGE_LOAD_FLOW_SETTINGS = CGM_RELAXED_3_CUSTOM
-NET_INTERCHANGE_THRESHOLD = 200
-OPEN_NON_RETAINED_SWITCHES_BETWEEN_TN = True
+OPDE_MODELS_ELK_INDEX = emfos-opde-models
+ENABLE_DYNAMIC_MERGE_SETTINGS = True
+MERGE_LOAD_FLOW_SETTINGS = CGM_DEFAULT
+MERGE_LOAD_FLOW_SETTINGS_PRIORITY = CGM_DEFAULT,CGM_RELAXED_1,CGM_RELAXED_2,CGM_RELAXED_3_CUSTOM
+REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION = True

config/cgm_worker/post_processing.properties

@@ -0,0 +1,5 @@
+[MAIN]
+NET_INTERCHANGE_THRESHOLD = 200
+FIX_INJECTION_ERRORS = True
+INJECTION_THRESHOLD = 0.1
+SMALL_ISLAND_SIZE = 10

config/cgm_worker/replacement.properties

@@ -1,3 +1,2 @@
 [MAIN]
-REPLACEMENT_CONFIG = config/cgm_worker/replacement_config.json
-QUERY_FILTER = now-2w
+REPLACEMENT_CONFIG = config/cgm_worker/replacement_config.json

config/cgm_worker/replacement_conf.json

@@ -2,10 +2,13 @@
         "configuration": {
             "validity_start": "2024-02-22T00:00:00Z",
             "validity_end": "2024-02-22T23:59:59Z",
-            "timeZone": "CET"
+            "time_zone": "CET"
         },
-        "timeHorizon": {
-            "Request_list": ["ID", "1D", "2D", "WK", "1D", "MO", "YR"]
+        "time_horizon": {
+            "request_list": ["ID", "1D", "2D", "WK", "1D", "MO", "YR"]
+        },
+        "replacement_length": {
+            "request_list": ["2w", "2w", "2w", "2w", "2w", "1m", "4m"]
         },
         "month_ahead": {
             "hours": ["07:30", "08:30", "09:30", "06:30", "05:30", "04:30"],

config/cgm_worker/scaler.properties

@@ -1,4 +1,4 @@
 [MAIN]
-MAX_ITERATION = 10
+MAX_ITERATION = 15
 BALANCE_THRESHOLD = 2
-DEBUG = True
+DEBUG = True

config/integrations/object_storage.properties

@@ -1,4 +1,3 @@
 [MAIN]
-ELASTIC_TASKS_INDEX = emfos-tasks
 ELASTIC_MODELS_INDEX = emfos-opde-models
 ELASTIC_SCHEDULES_INDEX = emfos-schedules

config/model_quality/model_quality.properties

@@ -0,0 +1,4 @@
+[MAIN]
+INPUT_RMQ_QUEUE = object-storage.models.quality
+ELK_QUALITY_INDEX = emfos-model-quality
+ELK_STATISTICS_INDEX = emfos-model-statistics

config/model_validator/model_validator.properties

@@ -1,9 +1,13 @@
 [MAIN]
 INPUT_RMQ_QUEUE = object-storage.models.validation
+OUTPUT_RMQ_EXCHANGE = emf-quality.test
 VALIDATION_ELK_INDEX = emfos-igm-validation
 METADATA_ELK_INDEX = emfos-opde-models
 ELK_ID_FROM_METADATA_FIELDS = opde:Id,data-source
 ELK_ID_HASHING = True
-VALIDATION_LOAD_FLOW_SETTINGS = CGM_RELAXED_2
+ENABLE_DYNAMIC_VALIDATION_SETTINGS = False
+VALIDATION_LOAD_FLOW_SETTINGS = IGM_VALIDATION
+VALIDATION_LOAD_FLOW_SETTINGS_PRIORITY = IGM_VALIDATION,CGM_DEFAULT,CGM_RELAXED_1,CGM_RELAXED_2,CGM_RELAXED_3_CUSTOM
 CHECK_NON_RETAINED_SWITCHES = False
-CHECK_KIRCHHOFF_FIRST_LAW = False
+CHECK_KIRCHHOFF_FIRST_LAW = False
+OPEN_NON_RETAINED_SWITCHES = True

config/task_generator/manual_task_generator.properties

@@ -14,12 +14,10 @@ TASK_REFERENCE_TIME =
 TASK_VERSION =
 TASK_MERGING_ENTITY =
 RUN_REPLACEMENT =
-RUN_REPLACEMENT_LOCAL =
 RUN_SCALING =
 UPLOAD_TO_OPDM =
 UPLOAD_TO_MINIO =
 SEND_MERGE_REPORT =
-PRE_TEMP_FIXES =
 POST_TEMP_FIXES =
 FORCE_OUTAGE_FIX =
 FIX_NET_INTERCHANGE2 =

config/task_generator/process_conf.json

@@ -312,6 +312,46 @@
         "send_merge_report": "True"
        }
       },
+      {
+       "@context": "https://example.com/schemas/run.jsonld",
+       "@id": "https://example.com/runs/IntraDayRMM/EOD",
+       "@type": "Run",
+       "process_id": "https://example.com/processes/RMM_CREATION",
+       "valid_from": "",
+       "valid_to": "",
+       "gate_open": "PT1H",
+       "gate_close": "PT45M",
+       "run_at": "00 15 29 02 3",
+       "time_frame": "ID",
+       "data_timestamps": "30 * * * *",
+       "data_resolution": "PT1H",
+       "tags": [],
+       "properties": {
+        "merge_type": "BA",
+        "merging_entity": "BALTICRCC",
+        "included": [
+         "AST",
+         "PSE",
+         "ELERING",
+	 "LITGRID"
+        ],
+        "excluded": [],
+        "local_import": [],
+        "time_horizon": "ID",
+        "version": "001",
+        "mas": "http://www.baltic-rsc.eu/OperationalPlanning/RMM",
+        "replacement": "True",
+        "replacement_local":"False",
+        "pre_temp_fixes":"True",
+        "post_temp_fixes":"True",
+        "fix_net_interchange2": "False",
+        "force_outage_fix":"False",
+        "scaling": "False",
+        "upload_to_opdm": "False",
+        "upload_to_minio": "True",
+        "send_merge_report": "True"
+       }
+      },
       {
        "@context": "https://example.com/schemas/run.jsonld",
        "@id": "https://example.com/runs/DayAheadRMM",

config/task_generator/task_generator.properties

@@ -1,6 +1,6 @@
 [MAIN]
 RMQ_EXCHANGE = emf-tasks
-TASK_ELK_INDEX =
+TASK_ELK_INDEX = emfos-tasks
 TASK_WINDOW_DURATION = PT1M
 TASK_WINDOW_REFERENCE = currentMinuteStart
 TIMESTAMP =

docker/Dockerfile

@@ -1,17 +1,17 @@
 # Define base container image (platform has to be defined if using with ARM64 architecture)
-FROM python:3.11 as emfos-base
-#FROM --platform=linux/amd64 python:3.11 as emfos-base
+FROM python:3.11-slim as emfos-base
+#FROM --platform=linux/amd64 python:3.11-slim as emfos-base
 
 # Set working directory
 WORKDIR /src
 
 # Install tools to set up python environment
+RUN apt-get update && apt-get install -y --no-install-recommends procps && rm -rf /var/lib/apt/lists/*
 RUN pip install --upgrade setuptools pip uv pipenv
 
 # Install needed modules
 COPY Pipfile .
 COPY Pipfile.lock .
-#COPY requirements.txt .
 #RUN PIPENV_VENV_IN_PROJECT=1 pipenv install --deploy
 #RUN pipenv install --system --deploy
 RUN pipenv requirements > requirements.txt
@@ -73,7 +73,6 @@ WORKDIR /src
 # Copy files to container
 COPY emf/model_merger/ ./emf/model_merger/
 COPY emf/common/ ./emf/common/
-COPY emf/task_generator/ ./emf/task_generator/
 COPY config/ ./config/
 
 # Start process
@@ -95,3 +94,15 @@ COPY config/ ./config/
 ENV PYTHONPATH="${PYTHONPATH}:/src"
 ENTRYPOINT ["python", "emf/task_generator/worker.py"]
 #ENTRYPOINT ["/bin/bash"]
+
+# Model Quality image
+FROM balticrcc/emfos-base:latest as emfos-model-quality
+
+WORKDIR /src
+
+COPY emf/model_quality/ ./emf/model_quality/
+COPY emf/common/ ./emf/common/
+COPY config/ ./config/
+
+ENV PYTHONPATH="${PYTHONPATH}:/src"
+ENTRYPOINT ["python", "emf/model_quality/worker.py"]

emf/common/decorators.py

@@ -5,6 +5,7 @@
 # Start logger
 logger = logging.getLogger(__name__)
 
+
 def performance_counter(units='seconds'):
     """Counts performance of the function"""
     def decorator_performance_counter(func):
@@ -15,7 +16,7 @@ def wrapper_performance_counter(*args, **kwargs):
             duration = round(time.perf_counter() - start_time, 2)
             if units == 'minutes':
                 duration = round(duration / 60, 2)  # counting by minutes
-            logger.info(f"Process {func.__name__!r} finished with duration: {duration} {units}", extra={'numeric_value': duration})
+            logger.info(f"Process {func.__name__!r} finished with duration: {duration} {units}")
             return response
         return wrapper_performance_counter
     return decorator_performance_counter

emf/common/helpers/cgmes.py

@@ -0,0 +1,53 @@
+import logging
+import json
+import pandas as pd
+import config
+from lxml import etree
+
+logger = logging.getLogger(__name__)
+
+
+def export_to_cgmes_zip(triplets: list):
+    namespace_map = {
+        "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
+        "cim": "http://iec.ch/TC57/2013/CIM-schema-cim16#",
+        "md": "http://iec.ch/TC57/61970-552/ModelDescription/1#",
+        "entsoe": "http://entsoe.eu/CIM/SchemaExtension/3/1#",
+    }
+
+    rdf_map = json.load(config.paths.cgm_worker.CGMES_v2_4_15_2014_08_07)
+
+    return pd.concat(triplets, ignore_index=True).export_to_cimxml(rdf_map=rdf_map,
+                                                                   namespace_map=namespace_map,
+                                                                   export_undefined=False,
+                                                                   export_type="xml_per_instance_zip_per_xml",
+                                                                   debug=False,
+                                                                   export_to_memory=True)
+
+
+def get_metadata_from_rdfxml(parsed_xml: etree._ElementTree):
+    """Parse model metadata form xml, returns a dictionary with metadata"""
+
+    assert isinstance(parsed_xml, etree._ElementTree)
+
+    rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+
+    header = parsed_xml.find("{*}FullModel")  # Version update proof, as long as element name remains the same
+    meta_elements = header.getchildren()
+
+    # Add model ID from FullModel@about
+    metadata = {"Model.mRID": header.attrib.get(f'{{{rdf}}}about').split(":")[-1]}
+
+    # Add all other metadata
+    for element in meta_elements:
+        key = element.tag.split("}")[1]
+        value = element.text if element.text else element.attrib.get(f"{{{rdf}}}resource")
+        if existing_value := metadata.get(key):
+            value = existing_value + [value] if isinstance(existing_value, list) else [existing_value, value]
+        metadata[key] = value
+
+    return metadata
+
+
+if __name__ == "__main__":
+    pass