From 3f7b2d59f9d3e6533ef9aec6fdd63a57ba2dda93 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Wed, 4 Jun 2025 15:37:53 +0300 Subject: [PATCH 01/67] added uv project dependency manager --- .python-version | 1 + pyproject.toml | 34 +++ uv.lock | 718 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 753 insertions(+) create mode 100644 .python-version create mode 100644 pyproject.toml create mode 100644 uv.lock diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..2c073331 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..98da0e93 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "emf" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "aniso8601>=10.0.1", + "croniter>=6.0.0", + "edx>=0.0.16", + "elasticsearch>=9.0.1", + "lxml>=5.4.0", + "minio>=7.2.15", + "ndjson>=0.3.1", + "opdm-api>=0.1.3", + "pandas==2.2.2", + "pika>=1.3.2", + "pypowsybl==1.9.0", + "pytz>=2025.2", + "requests>=2.32.3", + "saxonche>=12.7.0", + "triplets>=0.0.10", +] + +[[tool.uv.index]] +name = "pypi" +url = "https://pypi.org/simple" + +[[tool.uv.index]] +name = "rcc-pypi" +url = "https://artifactory.elering.sise/artifactory/api/pypi/rcc-pypi/simple" + +[tool.uv] +allow-insecure-host = ["artifactory.elering.sise"] diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..02506243 --- /dev/null +++ b/uv.lock @@ -0,0 +1,718 @@ +version = 1 +revision = 2 +requires-python = ">=3.11" +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version < '3.12'", +] + +[[package]] +name = "aniso8601" +version = "10.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/8d/52179c4e3f1978d3d9a285f98c706642522750ef343e9738286130423730/aniso8601-10.0.1.tar.gz", hash = "sha256:25488f8663dd1528ae1f54f94ac1ea51ae25b4d531539b8bc707fed184d16845", size = 47190, upload-time = "2025-04-18T17:29:42.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/75/e0e10dc7ed1408c28e03a6cb2d7a407f99320eb953f229d008a7a6d05546/aniso8601-10.0.1-py2.py3-none-any.whl", hash = "sha256:eb19717fd4e0db6de1aab06f12450ab92144246b257423fe020af5748c0cb89e", size = 52848, upload-time = "2025-04-18T17:29:41.492Z" }, +] + +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "21.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/e9/184b8ccce6683b0aa2fbb7ba5683ea4b9c5763f1356347f1312c32e3c66e/argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3", size = 1779911, upload-time = "2021-12-01T08:52:55.68Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/13/838ce2620025e9666aa8f686431f67a29052241692a3dd1ae9d3692a89d3/argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367", size = 29658, upload-time = "2021-12-01T09:09:17.016Z" }, + { url = "https://files.pythonhosted.org/packages/b3/02/f7f7bb6b6af6031edb11037639c697b912e1dea2db94d436e681aea2f495/argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9524464572e12979364b7d600abf96181d3541da11e23ddf565a32e70bd4dc0d", size = 80583, upload-time = "2021-12-01T09:09:19.546Z" }, + { url = "https://files.pythonhosted.org/packages/ec/f7/378254e6dd7ae6f31fe40c8649eea7d4832a42243acaf0f1fff9083b2bed/argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae", size = 86168, upload-time = "2021-12-01T09:09:21.445Z" }, + { url = "https://files.pythonhosted.org/packages/74/f6/4a34a37a98311ed73bb80efe422fed95f2ac25a4cacc5ae1d7ae6a144505/argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58ed19212051f49a523abb1dbe954337dc82d947fb6e5a0da60f7c8471a8476c", size = 82709, upload-time = "2021-12-01T09:09:18.182Z" }, + { url = "https://files.pythonhosted.org/packages/74/2b/73d767bfdaab25484f7e7901379d5f8793cccbb86c6e0cbc4c1b96f63896/argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bd46088725ef7f58b5a1ef7ca06647ebaf0eb4baff7d1d0d177c6cc8744abd86", size = 83613, upload-time = "2021-12-01T09:09:22.741Z" }, + { url = "https://files.pythonhosted.org/packages/4f/fd/37f86deef67ff57c76f137a67181949c2d408077e2e3dd70c6c42912c9bf/argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_i686.whl", hash = "sha256:8cd69c07dd875537a824deec19f978e0f2078fdda07fd5c42ac29668dda5f40f", size = 84583, upload-time = "2021-12-01T09:09:24.177Z" }, + { url = "https://files.pythonhosted.org/packages/6f/52/5a60085a3dae8fded8327a4f564223029f5f54b0cb0455a31131b5363a01/argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e", size = 88475, upload-time = "2021-12-01T09:09:26.673Z" }, + { url = "https://files.pythonhosted.org/packages/8b/95/143cd64feb24a15fa4b189a3e1e7efbaeeb00f39a51e99b26fc62fbacabd/argon2_cffi_bindings-21.2.0-cp36-abi3-win32.whl", hash = "sha256:603ca0aba86b1349b147cab91ae970c63118a0f30444d4bc80355937c950c082", size = 27698, upload-time = "2021-12-01T09:09:27.87Z" }, + { url = "https://files.pythonhosted.org/packages/37/2c/e34e47c7dee97ba6f01a6203e0383e15b60fb85d78ac9a15cd066f6fe28b/argon2_cffi_bindings-21.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:b2ef1c30440dbbcba7a5dc3e319408b59676e2e039e2ae11a8775ecf482b192f", size = 30817, upload-time = "2021-12-01T09:09:30.267Z" }, + { url = "https://files.pythonhosted.org/packages/5a/e4/bf8034d25edaa495da3c8a3405627d2e35758e44ff6eaa7948092646fdcc/argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93", size = 53104, upload-time = "2021-12-01T09:09:31.335Z" }, +] + +[[package]] +name = "attrs" +version = "25.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, +] + +[[package]] +name = "certifi" +version = "2025.4.26" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/9e/c05b3920a3b7d20d3d3310465f50348e5b3694f4f88c6daf736eef3024c4/certifi-2025.4.26.tar.gz", hash = "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6", size = 160705, upload-time = "2025-04-26T02:12:29.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/7e/3db2bd1b1f9e95f7cddca6d6e75e2f2bd9f51b1246e546d88addca0106bd/certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3", size = 159618, upload-time = "2025-04-26T02:12:27.662Z" }, +] + +[[package]] +name = "cffi" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, + { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, + { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, + { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, + { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, + { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, + { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, + { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, + { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, + { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, + { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, + { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, + { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, + { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, + { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, + { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, + { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, + { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, + { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, + { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/33/89c2ced2b67d1c2a61c19c6751aa8902d46ce3dacb23600a283619f5a12d/charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", size = 126367, upload-time = "2025-05-02T08:34:42.01Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/85/4c40d00dcc6284a1c1ad5de5e0996b06f39d8232f1031cd23c2f5c07ee86/charset_normalizer-3.4.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2", size = 198794, upload-time = "2025-05-02T08:32:11.945Z" }, + { url = "https://files.pythonhosted.org/packages/41/d9/7a6c0b9db952598e97e93cbdfcb91bacd89b9b88c7c983250a77c008703c/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645", size = 142846, upload-time = "2025-05-02T08:32:13.946Z" }, + { url = "https://files.pythonhosted.org/packages/66/82/a37989cda2ace7e37f36c1a8ed16c58cf48965a79c2142713244bf945c89/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd", size = 153350, upload-time = "2025-05-02T08:32:15.873Z" }, + { url = "https://files.pythonhosted.org/packages/df/68/a576b31b694d07b53807269d05ec3f6f1093e9545e8607121995ba7a8313/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8", size = 145657, upload-time = "2025-05-02T08:32:17.283Z" }, + { url = "https://files.pythonhosted.org/packages/92/9b/ad67f03d74554bed3aefd56fe836e1623a50780f7c998d00ca128924a499/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f", size = 147260, upload-time = "2025-05-02T08:32:18.807Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e6/8aebae25e328160b20e31a7e9929b1578bbdc7f42e66f46595a432f8539e/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7", size = 149164, upload-time = "2025-05-02T08:32:20.333Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f2/b3c2f07dbcc248805f10e67a0262c93308cfa149a4cd3d1fe01f593e5fd2/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9", size = 144571, upload-time = "2025-05-02T08:32:21.86Z" }, + { url = "https://files.pythonhosted.org/packages/60/5b/c3f3a94bc345bc211622ea59b4bed9ae63c00920e2e8f11824aa5708e8b7/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544", size = 151952, upload-time = "2025-05-02T08:32:23.434Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4d/ff460c8b474122334c2fa394a3f99a04cf11c646da895f81402ae54f5c42/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82", size = 155959, upload-time = "2025-05-02T08:32:24.993Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2b/b964c6a2fda88611a1fe3d4c400d39c66a42d6c169c924818c848f922415/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0", size = 153030, upload-time = "2025-05-02T08:32:26.435Z" }, + { url = "https://files.pythonhosted.org/packages/59/2e/d3b9811db26a5ebf444bc0fa4f4be5aa6d76fc6e1c0fd537b16c14e849b6/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5", size = 148015, upload-time = "2025-05-02T08:32:28.376Z" }, + { url = "https://files.pythonhosted.org/packages/90/07/c5fd7c11eafd561bb51220d600a788f1c8d77c5eef37ee49454cc5c35575/charset_normalizer-3.4.2-cp311-cp311-win32.whl", hash = "sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a", size = 98106, upload-time = "2025-05-02T08:32:30.281Z" }, + { url = "https://files.pythonhosted.org/packages/a8/05/5e33dbef7e2f773d672b6d79f10ec633d4a71cd96db6673625838a4fd532/charset_normalizer-3.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28", size = 105402, upload-time = "2025-05-02T08:32:32.191Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a4/37f4d6035c89cac7930395a35cc0f1b872e652eaafb76a6075943754f095/charset_normalizer-3.4.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7", size = 199936, upload-time = "2025-05-02T08:32:33.712Z" }, + { url = "https://files.pythonhosted.org/packages/ee/8a/1a5e33b73e0d9287274f899d967907cd0bf9c343e651755d9307e0dbf2b3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3", size = 143790, upload-time = "2025-05-02T08:32:35.768Z" }, + { url = "https://files.pythonhosted.org/packages/66/52/59521f1d8e6ab1482164fa21409c5ef44da3e9f653c13ba71becdd98dec3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a", size = 153924, upload-time = "2025-05-02T08:32:37.284Z" }, + { url = "https://files.pythonhosted.org/packages/86/2d/fb55fdf41964ec782febbf33cb64be480a6b8f16ded2dbe8db27a405c09f/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214", size = 146626, upload-time = "2025-05-02T08:32:38.803Z" }, + { url = "https://files.pythonhosted.org/packages/8c/73/6ede2ec59bce19b3edf4209d70004253ec5f4e319f9a2e3f2f15601ed5f7/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a", size = 148567, upload-time = "2025-05-02T08:32:40.251Z" }, + { url = "https://files.pythonhosted.org/packages/09/14/957d03c6dc343c04904530b6bef4e5efae5ec7d7990a7cbb868e4595ee30/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd", size = 150957, upload-time = "2025-05-02T08:32:41.705Z" }, + { url = "https://files.pythonhosted.org/packages/0d/c8/8174d0e5c10ccebdcb1b53cc959591c4c722a3ad92461a273e86b9f5a302/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981", size = 145408, upload-time = "2025-05-02T08:32:43.709Z" }, + { url = "https://files.pythonhosted.org/packages/58/aa/8904b84bc8084ac19dc52feb4f5952c6df03ffb460a887b42615ee1382e8/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c", size = 153399, upload-time = "2025-05-02T08:32:46.197Z" }, + { url = "https://files.pythonhosted.org/packages/c2/26/89ee1f0e264d201cb65cf054aca6038c03b1a0c6b4ae998070392a3ce605/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b", size = 156815, upload-time = "2025-05-02T08:32:48.105Z" }, + { url = "https://files.pythonhosted.org/packages/fd/07/68e95b4b345bad3dbbd3a8681737b4338ff2c9df29856a6d6d23ac4c73cb/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d", size = 154537, upload-time = "2025-05-02T08:32:49.719Z" }, + { url = "https://files.pythonhosted.org/packages/77/1a/5eefc0ce04affb98af07bc05f3bac9094513c0e23b0562d64af46a06aae4/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f", size = 149565, upload-time = "2025-05-02T08:32:51.404Z" }, + { url = "https://files.pythonhosted.org/packages/37/a0/2410e5e6032a174c95e0806b1a6585eb21e12f445ebe239fac441995226a/charset_normalizer-3.4.2-cp312-cp312-win32.whl", hash = "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c", size = 98357, upload-time = "2025-05-02T08:32:53.079Z" }, + { url = "https://files.pythonhosted.org/packages/6c/4f/c02d5c493967af3eda9c771ad4d2bbc8df6f99ddbeb37ceea6e8716a32bc/charset_normalizer-3.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e", size = 105776, upload-time = "2025-05-02T08:32:54.573Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/a93df3366ed32db1d907d7593a94f1fe6293903e3e92967bebd6950ed12c/charset_normalizer-3.4.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0", size = 199622, upload-time = "2025-05-02T08:32:56.363Z" }, + { url = "https://files.pythonhosted.org/packages/04/93/bf204e6f344c39d9937d3c13c8cd5bbfc266472e51fc8c07cb7f64fcd2de/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf", size = 143435, upload-time = "2025-05-02T08:32:58.551Z" }, + { url = "https://files.pythonhosted.org/packages/22/2a/ea8a2095b0bafa6c5b5a55ffdc2f924455233ee7b91c69b7edfcc9e02284/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e", size = 153653, upload-time = "2025-05-02T08:33:00.342Z" }, + { url = "https://files.pythonhosted.org/packages/b6/57/1b090ff183d13cef485dfbe272e2fe57622a76694061353c59da52c9a659/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1", size = 146231, upload-time = "2025-05-02T08:33:02.081Z" }, + { url = "https://files.pythonhosted.org/packages/e2/28/ffc026b26f441fc67bd21ab7f03b313ab3fe46714a14b516f931abe1a2d8/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c", size = 148243, upload-time = "2025-05-02T08:33:04.063Z" }, + { url = "https://files.pythonhosted.org/packages/c0/0f/9abe9bd191629c33e69e47c6ef45ef99773320e9ad8e9cb08b8ab4a8d4cb/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691", size = 150442, upload-time = "2025-05-02T08:33:06.418Z" }, + { url = "https://files.pythonhosted.org/packages/67/7c/a123bbcedca91d5916c056407f89a7f5e8fdfce12ba825d7d6b9954a1a3c/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0", size = 145147, upload-time = "2025-05-02T08:33:08.183Z" }, + { url = "https://files.pythonhosted.org/packages/ec/fe/1ac556fa4899d967b83e9893788e86b6af4d83e4726511eaaad035e36595/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b", size = 153057, upload-time = "2025-05-02T08:33:09.986Z" }, + { url = "https://files.pythonhosted.org/packages/2b/ff/acfc0b0a70b19e3e54febdd5301a98b72fa07635e56f24f60502e954c461/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff", size = 156454, upload-time = "2025-05-02T08:33:11.814Z" }, + { url = "https://files.pythonhosted.org/packages/92/08/95b458ce9c740d0645feb0e96cea1f5ec946ea9c580a94adfe0b617f3573/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b", size = 154174, upload-time = "2025-05-02T08:33:13.707Z" }, + { url = "https://files.pythonhosted.org/packages/78/be/8392efc43487ac051eee6c36d5fbd63032d78f7728cb37aebcc98191f1ff/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148", size = 149166, upload-time = "2025-05-02T08:33:15.458Z" }, + { url = "https://files.pythonhosted.org/packages/44/96/392abd49b094d30b91d9fbda6a69519e95802250b777841cf3bda8fe136c/charset_normalizer-3.4.2-cp313-cp313-win32.whl", hash = "sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7", size = 98064, upload-time = "2025-05-02T08:33:17.06Z" }, + { url = "https://files.pythonhosted.org/packages/e9/b0/0200da600134e001d91851ddc797809e2fe0ea72de90e09bec5a2fbdaccb/charset_normalizer-3.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980", size = 105641, upload-time = "2025-05-02T08:33:18.753Z" }, + { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626, upload-time = "2025-05-02T08:34:40.053Z" }, +] + +[[package]] +name = "croniter" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "pytz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/2f/44d1ae153a0e27be56be43465e5cb39b9650c781e001e7864389deb25090/croniter-6.0.0.tar.gz", hash = "sha256:37c504b313956114a983ece2c2b07790b1f1094fe9d81cc94739214748255577", size = 64481, upload-time = "2024-12-17T17:17:47.32Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/4b/290b4c3efd6417a8b0c284896de19b1d5855e6dbdb97d2a35e68fa42de85/croniter-6.0.0-py2.py3-none-any.whl", hash = "sha256:2f878c3856f17896979b2a4379ba1f09c83e374931ea15cc835c5dd2eee9b368", size = 25468, upload-time = "2024-12-17T17:17:45.359Z" }, +] + +[[package]] +name = "edx" +version = "0.0.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "requests" }, + { name = "urllib3" }, + { name = "zeep" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/35/87eeeeee86d8da9988faf99a77852dcb743c7b42408134dfa6a29bf0cefe/EDX-0.0.16.tar.gz", hash = "sha256:c48b98eb1e9c4343773631b436063c318c13948dda31b8f0691bcfa4052ae2c9", size = 20615, upload-time = "2024-01-03T08:07:16.679Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/4b/f7e4e3d0290cc7741877865fd8d9aaf1d18f8a1b1905477739f01043e69f/EDX-0.0.16-py3-none-any.whl", hash = "sha256:0a12bdf933c8f7ac63eb03e48b6a6442cda8ff9ae76b79eab57eb7ce36143d8a", size = 12521, upload-time = "2024-01-03T08:07:15.07Z" }, +] + +[[package]] +name = "elastic-transport" +version = "8.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/54/d498a766ac8fa475f931da85a154666cc81a70f8eb4a780bc8e4e934e9ac/elastic_transport-8.17.1.tar.gz", hash = "sha256:5edef32ac864dca8e2f0a613ef63491ee8d6b8cfb52881fa7313ba9290cac6d2", size = 73425, upload-time = "2025-03-13T07:28:30.776Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/cd/b71d5bc74cde7fc6fd9b2ff9389890f45d9762cbbbf81dc5e51fd7588c4a/elastic_transport-8.17.1-py3-none-any.whl", hash = "sha256:192718f498f1d10c5e9aa8b9cf32aed405e469a7f0e9d6a8923431dbb2c59fb8", size = 64969, upload-time = "2025-03-13T07:28:29.031Z" }, +] + +[[package]] +name = "elasticsearch" +version = "9.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "elastic-transport" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/58/0081e189ef83dd1f11cb600df842bffb8eaa05c097cb1672c5bd335b46c2/elasticsearch-9.0.1.tar.gz", hash = "sha256:76f9b519cfd15a860f615a94ba90dcb9543b721306bc5144ecc15f0b4d5d2781", size = 819519, upload-time = "2025-04-28T13:47:17.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/eb/03140387b1a378878a4228d0d796e80298ac54fc4fe6fe9e39087d142efe/elasticsearch-9.0.1-py3-none-any.whl", hash = "sha256:9fd110f9bb77310343709d9cccbd523a431a6e528d8441b01a26b97eef3238a3", size = 905510, upload-time = "2025-04-28T13:47:13.531Z" }, +] + +[[package]] +name = "emf" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "aniso8601" }, + { name = "croniter" }, + { name = "edx" }, + { name = "elasticsearch" }, + { name = "lxml" }, + { name = "minio" }, + { name = "ndjson" }, + { name = "opdm-api" }, + { name = "pandas" }, + { name = "pika" }, + { name = "pypowsybl" }, + { name = "pytz" }, + { name = "requests" }, + { name = "saxonche" }, + { name = "triplets" }, +] + +[package.metadata] +requires-dist = [ + { name = "aniso8601", specifier = ">=10.0.1" }, + { name = "croniter", specifier = ">=6.0.0" }, + { name = "edx", specifier = ">=0.0.16" }, + { name = "elasticsearch", specifier = ">=9.0.1" }, + { name = "lxml", specifier = ">=5.4.0" }, + { name = "minio", specifier = ">=7.2.15" }, + { name = "ndjson", specifier = ">=0.3.1" }, + { name = "opdm-api", specifier = ">=0.1.3" }, + { name = "pandas", specifier = "==2.2.2" }, + { name = "pika", specifier = ">=1.3.2" }, + { name = "pypowsybl", specifier = "==1.9.0" }, + { name = "pytz", specifier = ">=2025.2" }, + { name = "requests", specifier = ">=2.32.3" }, + { name = "saxonche", specifier = ">=12.7.0" }, + { name = "triplets", specifier = ">=0.0.10" }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, +] + +[[package]] +name = "isodate" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, +] + +[[package]] +name = "lxml" +version = "5.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/3d/14e82fc7c8fb1b7761f7e748fd47e2ec8276d137b6acfe5a4bb73853e08f/lxml-5.4.0.tar.gz", hash = "sha256:d12832e1dbea4be280b22fd0ea7c9b87f0d8fc51ba06e92dc62d52f804f78ebd", size = 3679479, upload-time = "2025-04-23T01:50:29.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/2d/67693cc8a605a12e5975380d7ff83020dcc759351b5a066e1cced04f797b/lxml-5.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:98a3912194c079ef37e716ed228ae0dcb960992100461b704aea4e93af6b0bb9", size = 8083240, upload-time = "2025-04-23T01:45:18.566Z" }, + { url = "https://files.pythonhosted.org/packages/73/53/b5a05ab300a808b72e848efd152fe9c022c0181b0a70b8bca1199f1bed26/lxml-5.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ea0252b51d296a75f6118ed0d8696888e7403408ad42345d7dfd0d1e93309a7", size = 4387685, upload-time = "2025-04-23T01:45:21.387Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/1a3879c5f512bdcd32995c301886fe082b2edd83c87d41b6d42d89b4ea4d/lxml-5.4.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b92b69441d1bd39f4940f9eadfa417a25862242ca2c396b406f9272ef09cdcaa", size = 4991164, upload-time = "2025-04-23T01:45:23.849Z" }, + { url = "https://files.pythonhosted.org/packages/f9/94/bbc66e42559f9d04857071e3b3d0c9abd88579367fd2588a4042f641f57e/lxml-5.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20e16c08254b9b6466526bc1828d9370ee6c0d60a4b64836bc3ac2917d1e16df", size = 4746206, upload-time = "2025-04-23T01:45:26.361Z" }, + { url = "https://files.pythonhosted.org/packages/66/95/34b0679bee435da2d7cae895731700e519a8dfcab499c21662ebe671603e/lxml-5.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7605c1c32c3d6e8c990dd28a0970a3cbbf1429d5b92279e37fda05fb0c92190e", size = 5342144, upload-time = "2025-04-23T01:45:28.939Z" }, + { url = "https://files.pythonhosted.org/packages/e0/5d/abfcc6ab2fa0be72b2ba938abdae1f7cad4c632f8d552683ea295d55adfb/lxml-5.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ecf4c4b83f1ab3d5a7ace10bafcb6f11df6156857a3c418244cef41ca9fa3e44", size = 4825124, upload-time = "2025-04-23T01:45:31.361Z" }, + { url = "https://files.pythonhosted.org/packages/5a/78/6bd33186c8863b36e084f294fc0a5e5eefe77af95f0663ef33809cc1c8aa/lxml-5.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cef4feae82709eed352cd7e97ae062ef6ae9c7b5dbe3663f104cd2c0e8d94ba", size = 4876520, upload-time = "2025-04-23T01:45:34.191Z" }, + { url = "https://files.pythonhosted.org/packages/3b/74/4d7ad4839bd0fc64e3d12da74fc9a193febb0fae0ba6ebd5149d4c23176a/lxml-5.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:df53330a3bff250f10472ce96a9af28628ff1f4efc51ccba351a8820bca2a8ba", size = 4765016, upload-time = "2025-04-23T01:45:36.7Z" }, + { url = "https://files.pythonhosted.org/packages/24/0d/0a98ed1f2471911dadfc541003ac6dd6879fc87b15e1143743ca20f3e973/lxml-5.4.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:aefe1a7cb852fa61150fcb21a8c8fcea7b58c4cb11fbe59c97a0a4b31cae3c8c", size = 5362884, upload-time = "2025-04-23T01:45:39.291Z" }, + { url = "https://files.pythonhosted.org/packages/48/de/d4f7e4c39740a6610f0f6959052b547478107967362e8424e1163ec37ae8/lxml-5.4.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ef5a7178fcc73b7d8c07229e89f8eb45b2908a9238eb90dcfc46571ccf0383b8", size = 4902690, upload-time = "2025-04-23T01:45:42.386Z" }, + { url = "https://files.pythonhosted.org/packages/07/8c/61763abd242af84f355ca4ef1ee096d3c1b7514819564cce70fd18c22e9a/lxml-5.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d2ed1b3cb9ff1c10e6e8b00941bb2e5bb568b307bfc6b17dffbbe8be5eecba86", size = 4944418, upload-time = "2025-04-23T01:45:46.051Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c5/6d7e3b63e7e282619193961a570c0a4c8a57fe820f07ca3fe2f6bd86608a/lxml-5.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:72ac9762a9f8ce74c9eed4a4e74306f2f18613a6b71fa065495a67ac227b3056", size = 4827092, upload-time = "2025-04-23T01:45:48.943Z" }, + { url = "https://files.pythonhosted.org/packages/71/4a/e60a306df54680b103348545706a98a7514a42c8b4fbfdcaa608567bb065/lxml-5.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f5cb182f6396706dc6cc1896dd02b1c889d644c081b0cdec38747573db88a7d7", size = 5418231, upload-time = "2025-04-23T01:45:51.481Z" }, + { url = "https://files.pythonhosted.org/packages/27/f2/9754aacd6016c930875854f08ac4b192a47fe19565f776a64004aa167521/lxml-5.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:3a3178b4873df8ef9457a4875703488eb1622632a9cee6d76464b60e90adbfcd", size = 5261798, upload-time = "2025-04-23T01:45:54.146Z" }, + { url = "https://files.pythonhosted.org/packages/38/a2/0c49ec6941428b1bd4f280650d7b11a0f91ace9db7de32eb7aa23bcb39ff/lxml-5.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e094ec83694b59d263802ed03a8384594fcce477ce484b0cbcd0008a211ca751", size = 4988195, upload-time = "2025-04-23T01:45:56.685Z" }, + { url = "https://files.pythonhosted.org/packages/7a/75/87a3963a08eafc46a86c1131c6e28a4de103ba30b5ae903114177352a3d7/lxml-5.4.0-cp311-cp311-win32.whl", hash = "sha256:4329422de653cdb2b72afa39b0aa04252fca9071550044904b2e7036d9d97fe4", size = 3474243, upload-time = "2025-04-23T01:45:58.863Z" }, + { url = "https://files.pythonhosted.org/packages/fa/f9/1f0964c4f6c2be861c50db380c554fb8befbea98c6404744ce243a3c87ef/lxml-5.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd3be6481ef54b8cfd0e1e953323b7aa9d9789b94842d0e5b142ef4bb7999539", size = 3815197, upload-time = "2025-04-23T01:46:01.096Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4c/d101ace719ca6a4ec043eb516fcfcb1b396a9fccc4fcd9ef593df34ba0d5/lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b5aff6f3e818e6bdbbb38e5967520f174b18f539c2b9de867b1e7fde6f8d95a4", size = 8127392, upload-time = "2025-04-23T01:46:04.09Z" }, + { url = "https://files.pythonhosted.org/packages/11/84/beddae0cec4dd9ddf46abf156f0af451c13019a0fa25d7445b655ba5ccb7/lxml-5.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:942a5d73f739ad7c452bf739a62a0f83e2578afd6b8e5406308731f4ce78b16d", size = 4415103, upload-time = "2025-04-23T01:46:07.227Z" }, + { url = "https://files.pythonhosted.org/packages/d0/25/d0d93a4e763f0462cccd2b8a665bf1e4343dd788c76dcfefa289d46a38a9/lxml-5.4.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:460508a4b07364d6abf53acaa0a90b6d370fafde5693ef37602566613a9b0779", size = 5024224, upload-time = "2025-04-23T01:46:10.237Z" }, + { url = "https://files.pythonhosted.org/packages/31/ce/1df18fb8f7946e7f3388af378b1f34fcf253b94b9feedb2cec5969da8012/lxml-5.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:529024ab3a505fed78fe3cc5ddc079464e709f6c892733e3f5842007cec8ac6e", size = 4769913, upload-time = "2025-04-23T01:46:12.757Z" }, + { url = "https://files.pythonhosted.org/packages/4e/62/f4a6c60ae7c40d43657f552f3045df05118636be1165b906d3423790447f/lxml-5.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ca56ebc2c474e8f3d5761debfd9283b8b18c76c4fc0967b74aeafba1f5647f9", size = 5290441, upload-time = "2025-04-23T01:46:16.037Z" }, + { url = "https://files.pythonhosted.org/packages/9e/aa/04f00009e1e3a77838c7fc948f161b5d2d5de1136b2b81c712a263829ea4/lxml-5.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a81e1196f0a5b4167a8dafe3a66aa67c4addac1b22dc47947abd5d5c7a3f24b5", size = 4820165, upload-time = "2025-04-23T01:46:19.137Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/e0b2f61fa2404bf0f1fdf1898377e5bd1b74cc9b2cf2c6ba8509b8f27990/lxml-5.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00b8686694423ddae324cf614e1b9659c2edb754de617703c3d29ff568448df5", size = 4932580, upload-time = "2025-04-23T01:46:21.963Z" }, + { url = "https://files.pythonhosted.org/packages/24/a2/8263f351b4ffe0ed3e32ea7b7830f845c795349034f912f490180d88a877/lxml-5.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c5681160758d3f6ac5b4fea370495c48aac0989d6a0f01bb9a72ad8ef5ab75c4", size = 4759493, upload-time = "2025-04-23T01:46:24.316Z" }, + { url = "https://files.pythonhosted.org/packages/05/00/41db052f279995c0e35c79d0f0fc9f8122d5b5e9630139c592a0b58c71b4/lxml-5.4.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:2dc191e60425ad70e75a68c9fd90ab284df64d9cd410ba8d2b641c0c45bc006e", size = 5324679, upload-time = "2025-04-23T01:46:27.097Z" }, + { url = "https://files.pythonhosted.org/packages/1d/be/ee99e6314cdef4587617d3b3b745f9356d9b7dd12a9663c5f3b5734b64ba/lxml-5.4.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:67f779374c6b9753ae0a0195a892a1c234ce8416e4448fe1e9f34746482070a7", size = 4890691, upload-time = "2025-04-23T01:46:30.009Z" }, + { url = "https://files.pythonhosted.org/packages/ad/36/239820114bf1d71f38f12208b9c58dec033cbcf80101cde006b9bde5cffd/lxml-5.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:79d5bfa9c1b455336f52343130b2067164040604e41f6dc4d8313867ed540079", size = 4955075, upload-time = "2025-04-23T01:46:32.33Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e1/1b795cc0b174efc9e13dbd078a9ff79a58728a033142bc6d70a1ee8fc34d/lxml-5.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d3c30ba1c9b48c68489dc1829a6eede9873f52edca1dda900066542528d6b20", size = 4838680, upload-time = "2025-04-23T01:46:34.852Z" }, + { url = "https://files.pythonhosted.org/packages/72/48/3c198455ca108cec5ae3662ae8acd7fd99476812fd712bb17f1b39a0b589/lxml-5.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1af80c6316ae68aded77e91cd9d80648f7dd40406cef73df841aa3c36f6907c8", size = 5391253, upload-time = "2025-04-23T01:46:37.608Z" }, + { url = "https://files.pythonhosted.org/packages/d6/10/5bf51858971c51ec96cfc13e800a9951f3fd501686f4c18d7d84fe2d6352/lxml-5.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4d885698f5019abe0de3d352caf9466d5de2baded00a06ef3f1216c1a58ae78f", size = 5261651, upload-time = "2025-04-23T01:46:40.183Z" }, + { url = "https://files.pythonhosted.org/packages/2b/11/06710dd809205377da380546f91d2ac94bad9ff735a72b64ec029f706c85/lxml-5.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aea53d51859b6c64e7c51d522c03cc2c48b9b5d6172126854cc7f01aa11f52bc", size = 5024315, upload-time = "2025-04-23T01:46:43.333Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b0/15b6217834b5e3a59ebf7f53125e08e318030e8cc0d7310355e6edac98ef/lxml-5.4.0-cp312-cp312-win32.whl", hash = "sha256:d90b729fd2732df28130c064aac9bb8aff14ba20baa4aee7bd0795ff1187545f", size = 3486149, upload-time = "2025-04-23T01:46:45.684Z" }, + { url = "https://files.pythonhosted.org/packages/91/1e/05ddcb57ad2f3069101611bd5f5084157d90861a2ef460bf42f45cced944/lxml-5.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1dc4ca99e89c335a7ed47d38964abcb36c5910790f9bd106f2a8fa2ee0b909d2", size = 3817095, upload-time = "2025-04-23T01:46:48.521Z" }, + { url = "https://files.pythonhosted.org/packages/87/cb/2ba1e9dd953415f58548506fa5549a7f373ae55e80c61c9041b7fd09a38a/lxml-5.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:773e27b62920199c6197130632c18fb7ead3257fce1ffb7d286912e56ddb79e0", size = 8110086, upload-time = "2025-04-23T01:46:52.218Z" }, + { url = "https://files.pythonhosted.org/packages/b5/3e/6602a4dca3ae344e8609914d6ab22e52ce42e3e1638c10967568c5c1450d/lxml-5.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ce9c671845de9699904b1e9df95acfe8dfc183f2310f163cdaa91a3535af95de", size = 4404613, upload-time = "2025-04-23T01:46:55.281Z" }, + { url = "https://files.pythonhosted.org/packages/4c/72/bf00988477d3bb452bef9436e45aeea82bb40cdfb4684b83c967c53909c7/lxml-5.4.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9454b8d8200ec99a224df8854786262b1bd6461f4280064c807303c642c05e76", size = 5012008, upload-time = "2025-04-23T01:46:57.817Z" }, + { url = "https://files.pythonhosted.org/packages/92/1f/93e42d93e9e7a44b2d3354c462cd784dbaaf350f7976b5d7c3f85d68d1b1/lxml-5.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cccd007d5c95279e529c146d095f1d39ac05139de26c098166c4beb9374b0f4d", size = 4760915, upload-time = "2025-04-23T01:47:00.745Z" }, + { url = "https://files.pythonhosted.org/packages/45/0b/363009390d0b461cf9976a499e83b68f792e4c32ecef092f3f9ef9c4ba54/lxml-5.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0fce1294a0497edb034cb416ad3e77ecc89b313cff7adbee5334e4dc0d11f422", size = 5283890, upload-time = "2025-04-23T01:47:04.702Z" }, + { url = "https://files.pythonhosted.org/packages/19/dc/6056c332f9378ab476c88e301e6549a0454dbee8f0ae16847414f0eccb74/lxml-5.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:24974f774f3a78ac12b95e3a20ef0931795ff04dbb16db81a90c37f589819551", size = 4812644, upload-time = "2025-04-23T01:47:07.833Z" }, + { url = "https://files.pythonhosted.org/packages/ee/8a/f8c66bbb23ecb9048a46a5ef9b495fd23f7543df642dabeebcb2eeb66592/lxml-5.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:497cab4d8254c2a90bf988f162ace2ddbfdd806fce3bda3f581b9d24c852e03c", size = 4921817, upload-time = "2025-04-23T01:47:10.317Z" }, + { url = "https://files.pythonhosted.org/packages/04/57/2e537083c3f381f83d05d9b176f0d838a9e8961f7ed8ddce3f0217179ce3/lxml-5.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e794f698ae4c5084414efea0f5cc9f4ac562ec02d66e1484ff822ef97c2cadff", size = 4753916, upload-time = "2025-04-23T01:47:12.823Z" }, + { url = "https://files.pythonhosted.org/packages/d8/80/ea8c4072109a350848f1157ce83ccd9439601274035cd045ac31f47f3417/lxml-5.4.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:2c62891b1ea3094bb12097822b3d44b93fc6c325f2043c4d2736a8ff09e65f60", size = 5289274, upload-time = "2025-04-23T01:47:15.916Z" }, + { url = "https://files.pythonhosted.org/packages/b3/47/c4be287c48cdc304483457878a3f22999098b9a95f455e3c4bda7ec7fc72/lxml-5.4.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:142accb3e4d1edae4b392bd165a9abdee8a3c432a2cca193df995bc3886249c8", size = 4874757, upload-time = "2025-04-23T01:47:19.793Z" }, + { url = "https://files.pythonhosted.org/packages/2f/04/6ef935dc74e729932e39478e44d8cfe6a83550552eaa072b7c05f6f22488/lxml-5.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1a42b3a19346e5601d1b8296ff6ef3d76038058f311902edd574461e9c036982", size = 4947028, upload-time = "2025-04-23T01:47:22.401Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f9/c33fc8daa373ef8a7daddb53175289024512b6619bc9de36d77dca3df44b/lxml-5.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4291d3c409a17febf817259cb37bc62cb7eb398bcc95c1356947e2871911ae61", size = 4834487, upload-time = "2025-04-23T01:47:25.513Z" }, + { url = "https://files.pythonhosted.org/packages/8d/30/fc92bb595bcb878311e01b418b57d13900f84c2b94f6eca9e5073ea756e6/lxml-5.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4f5322cf38fe0e21c2d73901abf68e6329dc02a4994e483adbcf92b568a09a54", size = 5381688, upload-time = "2025-04-23T01:47:28.454Z" }, + { url = "https://files.pythonhosted.org/packages/43/d1/3ba7bd978ce28bba8e3da2c2e9d5ae3f8f521ad3f0ca6ea4788d086ba00d/lxml-5.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:0be91891bdb06ebe65122aa6bf3fc94489960cf7e03033c6f83a90863b23c58b", size = 5242043, upload-time = "2025-04-23T01:47:31.208Z" }, + { url = "https://files.pythonhosted.org/packages/ee/cd/95fa2201041a610c4d08ddaf31d43b98ecc4b1d74b1e7245b1abdab443cb/lxml-5.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:15a665ad90054a3d4f397bc40f73948d48e36e4c09f9bcffc7d90c87410e478a", size = 5021569, upload-time = "2025-04-23T01:47:33.805Z" }, + { url = "https://files.pythonhosted.org/packages/2d/a6/31da006fead660b9512d08d23d31e93ad3477dd47cc42e3285f143443176/lxml-5.4.0-cp313-cp313-win32.whl", hash = "sha256:d5663bc1b471c79f5c833cffbc9b87d7bf13f87e055a5c86c363ccd2348d7e82", size = 3485270, upload-time = "2025-04-23T01:47:36.133Z" }, + { url = "https://files.pythonhosted.org/packages/fc/14/c115516c62a7d2499781d2d3d7215218c0731b2c940753bf9f9b7b73924d/lxml-5.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:bcb7a1096b4b6b24ce1ac24d4942ad98f983cd3810f9711bcd0293f43a9d8b9f", size = 3814606, upload-time = "2025-04-23T01:47:39.028Z" }, +] + +[[package]] +name = "minio" +version = "7.2.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi" }, + { name = "certifi" }, + { name = "pycryptodome" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/68/86a1cef80396e6a35a6fc4fafee5d28578c1a137bddd3ca2aa86f9b26a22/minio-7.2.15.tar.gz", hash = "sha256:5247df5d4dca7bfa4c9b20093acd5ad43e82d8710ceb059d79c6eea970f49f79", size = 138040, upload-time = "2025-01-19T08:57:26.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/6f/3690028e846fe432bfa5ba724a0dc37ec9c914965b7733e19d8ca2c4c48d/minio-7.2.15-py3-none-any.whl", hash = "sha256:c06ef7a43e5d67107067f77b6c07ebdd68733e5aa7eed03076472410ca19d876", size = 95075, upload-time = "2025-01-19T08:57:24.169Z" }, +] + +[[package]] +name = "ndjson" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/d5/209b6ca94566f9c94c0ec41cee1681c0a3b92a306a84a9b0fcd662088dc3/ndjson-0.3.1.tar.gz", hash = "sha256:bf9746cb6bb1cb53d172cda7f154c07c786d665ff28341e4e689b796b229e5d6", size = 6448, upload-time = "2020-02-25T05:01:07.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/c9/04ba0056011ba96a58163ebfd666d8385300bd12da1afe661a5a147758d7/ndjson-0.3.1-py2.py3-none-any.whl", hash = "sha256:839c22275e6baa3040077b83c005ac24199b94973309a8a1809be962c753a410", size = 5305, upload-time = "2020-02-25T05:01:06.39Z" }, +] + +[[package]] +name = "networkx" +version = "3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, +] + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963, upload-time = "2025-05-17T21:31:19.36Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743, upload-time = "2025-05-17T21:31:41.087Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616, upload-time = "2025-05-17T21:31:50.072Z" }, + { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579, upload-time = "2025-05-17T21:32:01.712Z" }, + { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005, upload-time = "2025-05-17T21:32:23.332Z" }, + { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570, upload-time = "2025-05-17T21:32:47.991Z" }, + { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548, upload-time = "2025-05-17T21:33:11.728Z" }, + { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521, upload-time = "2025-05-17T21:33:39.139Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866, upload-time = "2025-05-17T21:33:50.273Z" }, + { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455, upload-time = "2025-05-17T21:34:09.135Z" }, + { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" }, + { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" }, + { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" }, + { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" }, + { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" }, + { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" }, + { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828, upload-time = "2025-05-17T21:37:56.699Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006, upload-time = "2025-05-17T21:38:18.291Z" }, + { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765, upload-time = "2025-05-17T21:38:27.319Z" }, + { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736, upload-time = "2025-05-17T21:38:38.141Z" }, + { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719, upload-time = "2025-05-17T21:38:58.433Z" }, + { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072, upload-time = "2025-05-17T21:39:22.638Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213, upload-time = "2025-05-17T21:39:45.865Z" }, + { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632, upload-time = "2025-05-17T21:40:13.331Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532, upload-time = "2025-05-17T21:43:46.099Z" }, + { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885, upload-time = "2025-05-17T21:44:05.145Z" }, + { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467, upload-time = "2025-05-17T21:40:44Z" }, + { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144, upload-time = "2025-05-17T21:41:05.695Z" }, + { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217, upload-time = "2025-05-17T21:41:15.903Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014, upload-time = "2025-05-17T21:41:27.321Z" }, + { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935, upload-time = "2025-05-17T21:41:49.738Z" }, + { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122, upload-time = "2025-05-17T21:42:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143, upload-time = "2025-05-17T21:42:37.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260, upload-time = "2025-05-17T21:43:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225, upload-time = "2025-05-17T21:43:16.254Z" }, + { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374, upload-time = "2025-05-17T21:43:35.479Z" }, +] + +[[package]] +name = "opdm-api" +version = "0.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aniso8601" }, + { name = "lxml" }, + { name = "requests" }, + { name = "urllib3" }, + { name = "xmltodict" }, + { name = "zeep" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/2b/eef02b44c13013e66873094cf0da98731f34448c6628907d65669407cb9e/opdm-api-0.1.3.tar.gz", hash = "sha256:e5e1884d848190232ec62b7467e80451286a0c3c845faa87fbbf68837cc12c35", size = 26528, upload-time = "2025-05-21T11:09:26.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/95/15a9378972a1ac1a4ee2a7fee0fbb59ed443e72323c0cb7616ed539f52f2/opdm_api-0.1.3-py3-none-any.whl", hash = "sha256:91913afc4a74e5fdeea5ba4c3442382342e0031b6384dbb05568691ff9495ef2", size = 11497, upload-time = "2025-05-21T11:09:22.315Z" }, +] + +[[package]] +name = "pandas" +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/d9/ecf715f34c73ccb1d8ceb82fc01cd1028a65a5f6dbc57bfa6ea155119058/pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54", size = 4398391, upload-time = "2024-04-10T19:45:48.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/70/61704497903d43043e288017cb2b82155c0d41e15f5c17807920877b45c2/pandas-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288", size = 12574808, upload-time = "2024-04-10T19:44:35.516Z" }, + { url = "https://files.pythonhosted.org/packages/16/c6/75231fd47afd6b3f89011e7077f1a3958441264aca7ae9ff596e3276a5d0/pandas-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151", size = 11304876, upload-time = "2024-04-10T19:44:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/97/2d/7b54f80b93379ff94afb3bd9b0cd1d17b48183a0d6f98045bc01ce1e06a7/pandas-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b", size = 15602548, upload-time = "2024-04-10T19:44:42.902Z" }, + { url = "https://files.pythonhosted.org/packages/fc/a5/4d82be566f069d7a9a702dcdf6f9106df0e0b042e738043c0cc7ddd7e3f6/pandas-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee", size = 13031332, upload-time = "2024-04-10T19:44:46.98Z" }, + { url = "https://files.pythonhosted.org/packages/92/a2/b79c48f530673567805e607712b29814b47dcaf0d167e87145eb4b0118c6/pandas-2.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db", size = 16286054, upload-time = "2024-04-10T19:44:50.51Z" }, + { url = "https://files.pythonhosted.org/packages/40/c7/47e94907f1d8fdb4868d61bd6c93d57b3784a964d52691b77ebfdb062842/pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1", size = 13879507, upload-time = "2024-04-10T19:44:54.412Z" }, + { url = "https://files.pythonhosted.org/packages/ab/63/966db1321a0ad55df1d1fe51505d2cdae191b84c907974873817b0a6e849/pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24", size = 11634249, upload-time = "2024-04-10T19:44:58.183Z" }, + { url = "https://files.pythonhosted.org/packages/dd/49/de869130028fb8d90e25da3b7d8fb13e40f5afa4c4af1781583eb1ff3839/pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef", size = 12500886, upload-time = "2024-04-10T19:45:01.808Z" }, + { url = "https://files.pythonhosted.org/packages/db/7c/9a60add21b96140e22465d9adf09832feade45235cd22f4cb1668a25e443/pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce", size = 11340320, upload-time = "2024-04-11T18:36:14.398Z" }, + { url = "https://files.pythonhosted.org/packages/b0/85/f95b5f322e1ae13b7ed7e97bd999160fa003424711ab4dc8344b8772c270/pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad", size = 15204346, upload-time = "2024-04-10T19:45:05.903Z" }, + { url = "https://files.pythonhosted.org/packages/40/10/79e52ef01dfeb1c1ca47a109a01a248754ebe990e159a844ece12914de83/pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad", size = 12733396, upload-time = "2024-04-10T19:45:09.282Z" }, + { url = "https://files.pythonhosted.org/packages/35/9d/208febf8c4eb5c1d9ea3314d52d8bd415fd0ef0dd66bb24cc5bdbc8fa71a/pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76", size = 15858913, upload-time = "2024-04-10T19:45:12.514Z" }, + { url = "https://files.pythonhosted.org/packages/99/d1/2d9bd05def7a9e08a92ec929b5a4c8d5556ec76fae22b0fa486cbf33ea63/pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32", size = 13417786, upload-time = "2024-04-10T19:45:16.275Z" }, + { url = "https://files.pythonhosted.org/packages/22/a5/a0b255295406ed54269814bc93723cfd1a0da63fb9aaf99e1364f07923e5/pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23", size = 11498828, upload-time = "2024-04-10T19:45:19.85Z" }, +] + +[[package]] +name = "pika" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/db/d4102f356af18f316c67f2cead8ece307f731dd63140e2c71f170ddacf9b/pika-1.3.2.tar.gz", hash = "sha256:b2a327ddddf8570b4965b3576ac77091b850262d34ce8c1d8cb4e4146aa4145f", size = 145029, upload-time = "2023-05-05T14:25:43.368Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/f3/f412836ec714d36f0f4ab581b84c491e3f42c6b5b97a6c6ed1817f3c16d0/pika-1.3.2-py3-none-any.whl", hash = "sha256:0779a7c1fafd805672796085560d290213a465e4f6f76a6fb19e378d8041a14f", size = 155415, upload-time = "2023-05-05T14:25:41.484Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/8b/3c73abc9c759ecd3f1f7ceff6685840859e8070c4d947c93fae71f6a0bf2/platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc", size = 21362, upload-time = "2025-05-07T22:47:42.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" }, +] + +[[package]] +name = "prettytable" +version = "3.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/b1/85e18ac92afd08c533603e3393977b6bc1443043115a47bb094f3b98f94f/prettytable-3.16.0.tar.gz", hash = "sha256:3c64b31719d961bf69c9a7e03d0c1e477320906a98da63952bc6698d6164ff57", size = 66276, upload-time = "2025-03-24T19:39:04.008Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/c7/5613524e606ea1688b3bdbf48aa64bafb6d0a4ac3750274c43b6158a390f/prettytable-3.16.0-py3-none-any.whl", hash = "sha256:b5eccfabb82222f5aa46b798ff02a8452cf530a352c31bddfa29be41242863aa", size = 33863, upload-time = "2025-03-24T19:39:02.359Z" }, +] + +[[package]] +name = "pycparser" +version = "2.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, +] + +[[package]] +name = "pycryptodome" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" }, + { url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" }, + { url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" }, + { url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" }, + { url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" }, + { url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" }, + { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" }, + { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" }, + { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" }, + { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" }, + { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" }, + { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" }, + { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" }, + { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, +] + +[[package]] +name = "pypowsybl" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "networkx" }, + { name = "pandas" }, + { name = "prettytable" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/fc/3ed067e2bb12f11deb06eff55cc076d8cefbc50d3dd32fd84753c40291ce/pypowsybl-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f06d9007395b92802a1f7ce891de915265bcff8aa6ac8baca4a93436e091ca3", size = 62696454, upload-time = "2025-01-14T14:02:12.249Z" }, + { url = "https://files.pythonhosted.org/packages/2c/61/c50010a634daf2763223faec0f8813c9ae4e66fb142649ffb2210b33d9fc/pypowsybl-1.9.0-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:06a9ac540861f1bf271838c6151a61d35886b3f89ab23a29dc88d1f42392d6b6", size = 65553156, upload-time = "2025-01-14T14:03:06.163Z" }, + { url = "https://files.pythonhosted.org/packages/af/e0/fe0433cca394904263980af97c3718e1e01a2f6df7a3f358441dc534d09d/pypowsybl-1.9.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce052c2b2bcf8fd044cab613bd220806d6d74cce3b4cf56842f6d0aa35115579", size = 78710140, upload-time = "2025-01-14T14:04:01.942Z" }, + { url = "https://files.pythonhosted.org/packages/b5/fb/4fc0efac3ec83c9816c1d3c71f15f51b4997878470378b59755cc45371cb/pypowsybl-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:1015e4d0acc51634fe476a31a05e4b5827afe6cef2a5aa1a9b9ff97bba90c1a1", size = 61193440, upload-time = "2025-01-14T14:04:56.03Z" }, + { url = "https://files.pythonhosted.org/packages/53/ee/1e6bbf29a252f7da2382fb1d4b42b3a6c2dd51c3b263472185fdf9d46b83/pypowsybl-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:804b8a46989f303f5ef8851c2ba798fc7cf7b157b4ff3a9eb6a4e74bd117a5fa", size = 62708437, upload-time = "2025-01-14T14:02:24.731Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/1ef99b6d8ffed047f17a449094e04483bf8c3411b54ec5d3d9488fcf0056/pypowsybl-1.9.0-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:d76af6ee9ad031889e3356758c2d9e313c6cd5b625d6e72389a6c976e644f75a", size = 65555446, upload-time = "2025-01-14T14:03:19.976Z" }, + { url = "https://files.pythonhosted.org/packages/e3/41/b14fd11d372363db6e3f640ccc23949bf579e15da1ed0575dfffdc330694/pypowsybl-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9254f48c09e19fdef00f776f09c636976ee7373be7a2ef7209709993dadca40f", size = 78699091, upload-time = "2025-01-14T14:04:13.283Z" }, + { url = "https://files.pythonhosted.org/packages/c6/33/24f023ced3db9695637e4e14a2bda3a3875f423a376f9e0e74f0bb224664/pypowsybl-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:8728f39fdc94cc22bae4e4189f0875cf0e2d7b8711c012522322a784a6d6a2b1", size = 61202792, upload-time = "2025-01-14T14:05:05.966Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + +[[package]] +name = "requests" +version = "2.32.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218, upload-time = "2024-05-29T15:37:49.536Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928, upload-time = "2024-05-29T15:37:47.027Z" }, +] + +[[package]] +name = "requests-file" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/97/bf44e6c6bd8ddbb99943baf7ba8b1a8485bcd2fe0e55e5708d7fee4ff1ae/requests_file-2.1.0.tar.gz", hash = "sha256:0f549a3f3b0699415ac04d167e9cb39bccfb730cb832b4d20be3d9867356e658", size = 6891, upload-time = "2024-05-21T16:28:00.24Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/25/dd878a121fcfdf38f52850f11c512e13ec87c2ea72385933818e5b6c15ce/requests_file-2.1.0-py2.py3-none-any.whl", hash = "sha256:cf270de5a4c5874e84599fc5778303d496c10ae5e870bfa378818f35d21bda5c", size = 4244, upload-time = "2024-05-21T16:27:57.733Z" }, +] + +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, +] + +[[package]] +name = "saxonche" +version = "12.7.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/05/8c5961b4133a59aec24ce28f1127818ca99411ae2d8e46ddda32d54cb79a/saxonche-12.7.0-cp311-cp311-macosx_10_11_x86_64.whl", hash = "sha256:373c86dea41fed4455158e40ced27e0988caf5cf5c0529fc341dee446f580c37", size = 36157196, upload-time = "2025-05-16T08:55:25.782Z" }, + { url = "https://files.pythonhosted.org/packages/ec/aa/ee1385f8816925dd0c180f442b3edacfb5c3e5ed05ace06af34575cbb83d/saxonche-12.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:62d1315a2bf71d187da54d45046dbf37c6b84940aae63a700dc877287cfd9771", size = 34813749, upload-time = "2025-05-16T08:55:34.801Z" }, + { url = "https://files.pythonhosted.org/packages/e1/a4/834d3b07addc8c4a9619c0f0e7e56070c555f6bb302c4ef3f95589693ff5/saxonche-12.7.0-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:45abcb429647fd32758a3f572a0dbbcc6ec5d38d36868796b1f83a6e168630d6", size = 37764250, upload-time = "2025-05-16T08:55:43.42Z" }, + { url = "https://files.pythonhosted.org/packages/21/61/2beb2e9dd3d0aa63ee222f20ed415942c31a9dd162a0b7a973b1ad9ddb8f/saxonche-12.7.0-cp311-cp311-manylinux_2_24_x86_64.whl", hash = "sha256:262c284f772cad3a18dc06b0d55d4032849083c00049fe24c1e7e67a0c625bff", size = 38768813, upload-time = "2025-05-16T08:55:52.363Z" }, + { url = "https://files.pythonhosted.org/packages/cc/7f/9fd91e97b95f8dd0b0b2cd97cacd512dd359323d65319660833585f6b9c8/saxonche-12.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:36444ebf37f825f92c7844d0adb7c98c950b60072dbd305771c0df473c469003", size = 35594208, upload-time = "2025-05-16T08:56:01.121Z" }, + { url = "https://files.pythonhosted.org/packages/df/19/4151e2c88f55c094f69144acba3608dc1e188897fd4e679400844933a835/saxonche-12.7.0-cp312-cp312-macosx_10_11_x86_64.whl", hash = "sha256:e8d255498a49055a950ee999a84dee003e74dc4338f25806b1e274fb63f2a4ba", size = 36162700, upload-time = "2025-05-16T08:56:14.914Z" }, + { url = "https://files.pythonhosted.org/packages/55/26/98d18ea61e168630272a4fd3f9d350684bcfab8b2f969a1001ad3b7a6e10/saxonche-12.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:285a09e39f0bae143b304da55254d18cc5206fd8ef938822cec5947be299adb5", size = 34821331, upload-time = "2025-05-16T08:56:23.37Z" }, + { url = "https://files.pythonhosted.org/packages/3b/96/78434940683280d0882b53ee7fe9a197f327b05432519d4f25b939c1989a/saxonche-12.7.0-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:e861dc5eee567a28aafe0a1054585f3e8e9d1bdc9fe72dbddd756e34462c4251", size = 37691891, upload-time = "2025-05-16T08:56:32.203Z" }, + { url = "https://files.pythonhosted.org/packages/35/fe/688e288befa3e106aae30d2f1d6fc3080b492b9cb22f461a63cf6e05c573/saxonche-12.7.0-cp312-cp312-manylinux_2_24_x86_64.whl", hash = "sha256:fa55ba65344534652498ee6a1de86f32c2a40d273ec48eb996e62be45b9594c2", size = 38728204, upload-time = "2025-05-16T08:56:41.159Z" }, + { url = "https://files.pythonhosted.org/packages/80/a6/ca4177e7901c25572c6a32c1a5dbf842537745278ba765fb3f43943acbef/saxonche-12.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:b2729d254cf023b08aea5bd693c052b2e67dc8112ec009429ccae6eec0114002", size = 35595343, upload-time = "2025-05-16T08:56:49.556Z" }, + { url = "https://files.pythonhosted.org/packages/25/d4/27adf067fe1f171d4f84a88b31a217d9d721736d28564f8b252f4a5bfcdd/saxonche-12.7.0-cp313-cp313-macosx_10_11_x86_64.whl", hash = "sha256:5244ed00c59528539a7635ae7321f3b5b1b9866660621e7a0bc7036c31a79e4e", size = 36162668, upload-time = "2025-05-16T08:56:57.202Z" }, + { url = "https://files.pythonhosted.org/packages/d8/1b/346d78712dd6ba24e0906d9a2feb705d97111763a273528679ea79762ac8/saxonche-12.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d6cf1c7d2ead0241e6305b8a2c63098bd9e1060960b9ba687bd9daf12ebc1e9d", size = 34819960, upload-time = "2025-05-16T08:57:04.129Z" }, + { url = "https://files.pythonhosted.org/packages/a3/89/5bca5a5a5e9f9df769787bbaa22f9ba9a7eedd52748a2c27099dd500f421/saxonche-12.7.0-cp313-cp313-manylinux_2_24_aarch64.whl", hash = "sha256:12dc57c1ea8df7565d0e304f59edb7ea30cec808305259e7e866c633ef151400", size = 37686292, upload-time = "2025-05-16T08:57:11.771Z" }, + { url = "https://files.pythonhosted.org/packages/11/9f/e40415cea669d397a2a00f6bb1734b7030c39b2609fb3bd4c43d61bf3994/saxonche-12.7.0-cp313-cp313-manylinux_2_24_x86_64.whl", hash = "sha256:b0493b4360ce2d0a2b0d4a493f0027650dd183bd504e837d3848c084a30e59be", size = 38724250, upload-time = "2025-05-16T08:57:19.234Z" }, + { url = "https://files.pythonhosted.org/packages/b0/54/fffafa4c7a23908f698f65b694817c03139c1337faf213cba6206c79400b/saxonche-12.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:679a21212cc7a9027783121253c7039ae335c21cbc851dddff9f57778abbfc4d", size = 35594883, upload-time = "2025-05-16T08:57:26.203Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "triplets" +version = "0.0.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aniso8601" }, + { name = "lxml" }, + { name = "pandas" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/34/518c5c305bbd7bda58be7c9292c8ff9ff378894f15fb1666652f294ffe31/triplets-0.0.10.tar.gz", hash = "sha256:edf08b458a7afc2001e0db71e0efa6198363bdeba6f4b07ef46fe99a292687f7", size = 36572, upload-time = "2025-05-19T08:46:31.291Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/f2/328b54f0a2aec2311fa0d955a5b17e2acc2177a0e085b052b2098dde908e/triplets-0.0.10-py3-none-any.whl", hash = "sha256:2ab66c9036a20a741cccbf1170fa446947ded88665736cab8f7f1c2a8d0f6235", size = 21441, upload-time = "2025-05-19T08:46:28.127Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/bc/51647cd02527e87d05cb083ccc402f93e441606ff1f01739a62c8ad09ba5/typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4", size = 107423, upload-time = "2025-06-02T14:52:11.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/e0/552843e0d356fbb5256d21449fa957fa4eff3bbc135a74a691ee70c7c5da/typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af", size = 43839, upload-time = "2025-06-02T14:52:10.026Z" }, +] + +[[package]] +name = "tzdata" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, +] + +[[package]] +name = "urllib3" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/78/16493d9c386d8e60e442a35feac5e00f0913c0f4b7c217c11e8ec2ff53e0/urllib3-2.4.0.tar.gz", hash = "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466", size = 390672, upload-time = "2025-04-10T15:23:39.232Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/11/cc635220681e93a0183390e26485430ca2c7b5f9d33b15c74c2861cb8091/urllib3-2.4.0-py3-none-any.whl", hash = "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813", size = 128680, upload-time = "2025-04-10T15:23:37.377Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.2.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5", size = 101301, upload-time = "2024-01-06T02:10:57.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166, upload-time = "2024-01-06T02:10:55.763Z" }, +] + +[[package]] +name = "xmltodict" +version = "0.14.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/50/05/51dcca9a9bf5e1bce52582683ce50980bcadbc4fa5143b9f2b19ab99958f/xmltodict-0.14.2.tar.gz", hash = "sha256:201e7c28bb210e374999d1dde6382923ab0ed1a8a5faeece48ab525b7810a553", size = 51942, upload-time = "2024-10-16T06:10:29.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/45/fc303eb433e8a2a271739c98e953728422fa61a3c1f36077a49e395c972e/xmltodict-0.14.2-py2.py3-none-any.whl", hash = "sha256:20cc7d723ed729276e808f26fb6b3599f786cbc37e06c65e192ba77c40f20aac", size = 9981, upload-time = "2024-10-16T06:10:27.649Z" }, +] + +[[package]] +name = "zeep" +version = "4.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "isodate" }, + { name = "lxml" }, + { name = "platformdirs" }, + { name = "pytz" }, + { name = "requests" }, + { name = "requests-file" }, + { name = "requests-toolbelt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/35/4a91181499a1e098cb01e04a26a053714adf2fb1c25b40fdc5f46cfe2e4f/zeep-4.3.1.tar.gz", hash = "sha256:f45385e9e1b09d5550e0f51ab9fa7c6842713cab7194139372fd82a99c56a06e", size = 166468, upload-time = "2024-10-16T05:31:30.775Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/92/8a76eeccb5c176ea3de3965598b6fcd10dc9d72a560e59919b52845327ed/zeep-4.3.1-py3-none-any.whl", hash = "sha256:a637aa7eedb6330bb27e8c94c5233ddf23553904323adf9398f8cf5025acb216", size = 101690, upload-time = "2024-10-16T05:31:28.702Z" }, +] From 3c35ffd36dfca4140e7f85819289efdef4505e09 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Thu, 5 Jun 2025 14:52:54 +0300 Subject: [PATCH 02/67] bugfix: OPDM api fix to comply with release 11 due to changed structure of query response --- emf/common/integrations/opdm.py | 95 +++++++++++++++++---------------- 1 file changed, 48 insertions(+), 47 deletions(-) diff --git a/emf/common/integrations/opdm.py b/emf/common/integrations/opdm.py index 219b5903..511ec640 100644 --- a/emf/common/integrations/opdm.py +++ b/emf/common/integrations/opdm.py @@ -18,76 +18,77 @@ class OPDM(opdm_api.create_client): def __init__(self, server=OPDM_SERVER, username=OPDM_USERNAME, password=OPDM_PASSWORD, debug=False, verify=False): super().__init__(server, username, password, debug, verify) - # TODO - update query function in OPDM module - def query(self, object_type, meta=None): + def query(self, object_type, meta = None): - logger.info(f"Sending query to OPDM for {object_type} with parameters {meta}") + logger.info(f"Sending query to OPDM for {object_type} with parameters: {meta}") if meta is None: meta = {} raw_response = self.query_object(object_type, meta) - response = raw_response['sm:QueryResult']['sm:part'][1:] - - if type(response) == str: + if raw_response.get('sm:OperationFailure'): + logger.error(f"OPDM responded with error: {raw_response['sm:OperationFailure']['sm:part']}") response = [] - - logger.info(f"Number of responses: {len(response)} for query") + else: + # Extract metadata objects from response + response = raw_response['sm:QueryResult']['sm:part'][4:] + if isinstance(response, str): + response = [] + logger.info(f"Number of responses for query: {len(response)}") return response - def download_object(self, opdm_object, output_format='bytes', output_dir=None): + def download_object(self, opdm_object: dict): - if output_format == 'file': - for cimxml_file in opdm_object['opde:Component']: - file_id = cimxml_file['opdm:Profile']['opde:Id'] - file_name = cimxml_file['opdm:Profile']['pmd:fileName'] - file_path = os.path.join(output_dir, file_name) + model_meta = opdm_object + party = model_meta.get('pmd:modelPartReference', model_meta.get('pmd:TSO', '')) - logger.info(f"Downloading {file_name} with ID -> {file_id}") - response = self.get_content(file_id, return_payload=True) + for pos, model_part in enumerate(model_meta['opde:Component']): + model_part_meta = model_part['opdm:Profile'] + model_part_name = model_part_meta['pmd:fileName'] - with open(file_path, 'wb') as file_object: - message64_bytes = response['sm:GetContentResult']['sm:part'][1]['opdm:Profile']['opde:Content'].encode() - file_object.write(base64.b64decode(message64_bytes)) + # Maybe the file is all ready there (if OPDM subscription is enabled) + content_data = self.get_file(model_part_name) - logger.info(f"Saved to {file_path}") + # If file is not available on local client, lets request it from OPDM and download it + # More optimal, downloads whole model first (4 files) + if not content_data: + logger.warning("File not present on local client, requesting from OPDM service the whole Model") + content_meta = self.get_content(model_meta['opde:Id'], object_type="model") + content_data = self.get_file(model_part_name) - return None + # Less optimal, downloads each file separately + if not content_data: + logger.warning("File not present on local client, requesting from OPDM service the specific File") + content_meta = self.get_content(model_part_meta['opde:Id']) + content_data = self.get_file(model_part_name) - if output_format == 'bytes': - model_meta = opdm_object - party = model_meta.get('pmd:modelPartReference', model_meta.get('pmd:TSO', '')) + # If no data available set to None + if not content_data: + logger.error(f"{model_part_name} not available on webdav") + opdm_object['opde:Component'][pos]['opdm:Profile']["DATA"] = None - for pos, model_part in enumerate(model_meta['opde:Component']): - model_part_meta = model_part['opdm:Profile'] - model_part_name = model_part_meta['pmd:fileName'] + # Save data to metadata object + opdm_object['opde:Component'][pos]['opdm:Profile']["DATA"] = content_data - # Maybe the file is all ready there (if OPDM subscription is enabled) - content_data = self.get_file(model_part_name) + return opdm_object - # If file is not available on local client, lets request it from OPDM and download it + def download_object_to_filesystem(self, opdm_object: dict, output_dir: str | None = None): - # More optimal, downloads whole model first (4 files) - if not content_data: - logger.warning("File not present on local client, requesting from OPDM service the whole Model") - content_meta = self.get_content(model_meta['opde:Id'], object_type="model") - content_data = self.get_file(model_part_name) + for cimxml_file in opdm_object['opdm:OPDMObject']['opde:Component']: + file_id = cimxml_file['opdm:Profile']['opde:Id'] + file_name = cimxml_file['opdm:Profile']['pmd:fileName'] + file_path = os.path.join(output_dir, file_name) - # Less optimal, downloads each file separately - if not content_data: - logger.warning("File not present on local client, requesting from OPDM service the specific File") - content_meta = self.get_content(model_part_meta['opde:Id']) - content_data = self.get_file(model_part_name) + logger.info(f"Downloading {file_name} with ID: {file_id}") + response = self.get_content(file_id, return_payload=True) - # If no data available set to None - if not content_data: - logger.error(f"{model_part_name} not available on webdav") - opdm_object['opde:Component'][pos]['opdm:Profile']["DATA"] = None + with open(file_path, 'wb') as file_object: + message64_bytes = response['sm:GetContentResult']['sm:part'][4]['opdm:Profile']['opde:Content'].encode() + file_object.write(base64.b64decode(message64_bytes)) - # Save data to metadata object - opdm_object['opde:Component'][pos]['opdm:Profile']["DATA"] = content_data + logger.info(f"File saved to: {file_path}") - return opdm_object + return None def get_file(self, file_id): From 583e8b294cda9c5ff4ee5a3007d4155b021e3d7b Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Thu, 5 Jun 2025 14:48:33 +0300 Subject: [PATCH 03/67] #312 Quality module initial implementation --- config/cgm_worker/merger.properties | 1 + config/model_quality/model_quality.properties | 3 + .../model_validator.properties | 3 +- emf/model_merger/merge_functions.py | 2 +- emf/model_merger/model_merger.py | 40 +-- emf/model_merger/temporary_fixes.py | 4 +- emf/model_quality/model_quality.py | 229 ++++++++++++++++++ emf/model_quality/quality_functions.py | 0 emf/model_quality/worker.py | 20 ++ emf/model_validator/model_validator.py | 25 +- 10 files changed, 302 insertions(+), 25 deletions(-) create mode 100644 config/model_quality/model_quality.properties create mode 100644 emf/model_quality/model_quality.py create mode 100644 emf/model_quality/quality_functions.py create mode 100644 emf/model_quality/worker.py diff --git a/config/cgm_worker/merger.properties b/config/cgm_worker/merger.properties index 1e636b23..9b5fd173 100644 --- a/config/cgm_worker/merger.properties +++ b/config/cgm_worker/merger.properties @@ -5,6 +5,7 @@ INPUT_MINIO_FOLDER = IGM OUTPUT_MINIO_BUCKET = opde-confidential-models OUTPUT_MINIO_FOLDER = EMFOS/RMM MERGE_REPORT_ELK_INDEX = emfos-merge-reports +RABBIT_QUALITY_EXCHANGE = emf-quality.test SMALL_ISLAND_SIZE = 10 MERGE_LOAD_FLOW_SETTINGS = CGM_RELAXED_3_CUSTOM NET_INTERCHANGE_THRESHOLD = 200 diff --git a/config/model_quality/model_quality.properties b/config/model_quality/model_quality.properties new file mode 100644 index 00000000..263a766e --- /dev/null +++ b/config/model_quality/model_quality.properties @@ -0,0 +1,3 @@ +[MAIN] +INPUT_RMQ_QUEUE = model-quality.test +ELK_QUALITY_INDEX = test-emf-model-quality \ No newline at end of file diff --git a/config/model_validator/model_validator.properties b/config/model_validator/model_validator.properties index d2d0ed35..ec606b8e 100644 --- a/config/model_validator/model_validator.properties +++ b/config/model_validator/model_validator.properties @@ -6,4 +6,5 @@ ELK_ID_FROM_METADATA_FIELDS = opde:Id,data-source ELK_ID_HASHING = True VALIDATION_LOAD_FLOW_SETTINGS = CGM_RELAXED_2 CHECK_NON_RETAINED_SWITCHES = False -CHECK_KIRCHHOFF_FIRST_LAW = False \ No newline at end of file +CHECK_KIRCHHOFF_FIRST_LAW = False +RABBIT_QUALITY_EXCHANGE = emf-quality.test \ No newline at end of file diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index ac9ee348..a1a4c41f 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -284,7 +284,7 @@ def create_sv_and_updated_ssh(merged_model, original_models, models_as_triplets, filename_mask = "{scenarioTime:%Y%m%dT%H%MZ}_{processType}_{mergingEntity}-{domain}-{forEntity}_{messageType}_{version:03d}" ssh_data = triplets.cgmes_tools.update_filename_from_FullModel(ssh_data, filename_mask=filename_mask) - return sv_data, ssh_data + return sv_data, ssh_data, opdm_object_meta def fix_sv_shunts(sv_data, models_as_triplets): diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index d80e5574..d7ecf511 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -13,7 +13,7 @@ from io import BytesIO from zipfile import ZipFile from emf.common.config_parser import parse_app_properties -from emf.common.integrations import opdm, minio_api, elastic +from emf.common.integrations import opdm, minio_api, elastic, rabbit from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download from emf.common.integrations.object_storage.schedules import query_acnp_schedules, query_hvdc_schedules from emf.common.loadflow_tool import loadflow_settings @@ -182,9 +182,8 @@ def handle(self, task_object: dict, properties: dict, **kwargs): 'time_horizon': model['pmd:timeHorizon'], 'scenario_timestamp': model['pmd:scenarioDate']} for model in replacement_models_local] merged_model.replaced_entity.extend(replaced_entities_local) - - additional_models_data = additional_models_data + replacement_models_local - + #TODO change, keeping this to keep consistent naming structure for now + additional_models_data = replacement_models_local except Exception as error: logger.error(f"Failed to run replacement: {error} {error.with_traceback()}") else: @@ -293,14 +292,10 @@ def handle(self, task_object: dict, properties: dict, **kwargs): # Set default time horizon and scenario timestamp if not provided if not schedule_time_horizon or schedule_time_horizon == "AUTO": schedule_time_horizon = time_horizon - else: - logger.warning(f"Using replaced schedules from timehorizon: {schedule_time_horizon}") - + if not schedule_start: schedule_start = scenario_datetime - else: - logger.warning(f"Replaced schedules from schedule_start: {schedule_start}") - + # Get aligned schedules ac_schedules = query_acnp_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) dc_schedules = query_hvdc_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) @@ -331,13 +326,17 @@ def handle(self, task_object: dict, properties: dict, **kwargs): # Run post-processing post_p_start = datetime.datetime.now(datetime.UTC) - sv_data, ssh_data = run_post_merge_processing(input_models=input_models, + sv_data, ssh_data, opdm_object_meta = run_post_merge_processing(input_models=input_models, merged_model=merged_model, task_properties=task_properties, small_island_size=SMALL_ISLAND_SIZE, enable_temp_fixes=post_temp_fixes, time_horizon=new_time_horizon) + opdm_object_meta['pmd:content-reference'] = '' + opdm_object_meta['opde:Object-Type'] = '' + + # Package both input models and exported CGM profiles to in memory zip files serialized_data = merge_functions.export_to_cgmes_zip([ssh_data, sv_data]) post_p_end = datetime.datetime.now(datetime.UTC) @@ -421,6 +420,15 @@ def handle(self, task_object: dict, properties: dict, **kwargs): except Exception as error: logger.error(f"Failed to create merge report: {error}") + # Send model metadata to quality assurance module + if model_upload_to_minio and model_merge_report_send_to_elk: + try: + merge_report['minio-bucket'] = OUTPUT_MINIO_BUCKET + rabbit.BlockingClient().publish(payload=json.dumps(merge_report), exchange_name=RABBIT_QUALITY_EXCHANGE, + headers={"opde:Object-Type": "CGM"}) + except: + logger.error(f"Sending metadata to RabbitMQ failed: {error}") + # Stop Trace self.elk_logging_handler.stop_trace() @@ -459,13 +467,13 @@ def handle(self, task_object: dict, properties: dict, **kwargs): "job_period_start": "2024-05-24T22:00:00+00:00", "job_period_end": "2024-05-25T06:00:00+00:00", "task_properties": { - "timestamp_utc": "2025-04-29T22:30:00+00:00", + "timestamp_utc": "2025-06-01T12:30:00+00:00", "merge_type": "BA", "merging_entity": "BALTICRCC", - "included": ["AST"], + "included": ["LITGRID"], "excluded": [], - "local_import": ["LITGRID"], - "time_horizon": "ID", + "local_import": [], + "time_horizon": "2D", "version": "00", "mas": "http://www.baltic-rsc.eu/OperationalPlanning", "pre_temp_fixes": "True", @@ -475,7 +483,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): "replacement_local": "True", "scaling": "True", "upload_to_opdm": "False", - "upload_to_minio": "False", + "upload_to_minio": "True", "send_merge_report": "False", "force_outage_fix": "False", } diff --git a/emf/model_merger/temporary_fixes.py b/emf/model_merger/temporary_fixes.py index 0b13bfda..a74e9d76 100644 --- a/emf/model_merger/temporary_fixes.py +++ b/emf/model_merger/temporary_fixes.py @@ -291,7 +291,7 @@ def run_post_merge_processing(input_models: list, version = task_properties["version"] models_as_triplets = load_opdm_data(input_models) - sv_data, ssh_data = create_sv_and_updated_ssh(merged_model, input_models, models_as_triplets, + sv_data, ssh_data, opdm_object_meta = create_sv_and_updated_ssh(merged_model, input_models, models_as_triplets, scenario_datetime, time_horizon, version, merging_area, merging_entity, mas) @@ -334,7 +334,7 @@ def run_post_merge_processing(input_models: list, except KeyError: logger.warning(f"No fields for netInterchange") - return sv_data, ssh_data + return sv_data, ssh_data, opdm_object_meta def fix_model_outages(merged_model: object, tso_list: list, scenario_datetime: str, time_horizon: str, debug: bool = False): diff --git a/emf/model_quality/model_quality.py b/emf/model_quality/model_quality.py new file mode 100644 index 00000000..7be4e2b3 --- /dev/null +++ b/emf/model_quality/model_quality.py @@ -0,0 +1,229 @@ +import logging +import pandas as pd +import config +import json +import pypowsybl as pp +from emf.common.loadflow_tool.helper import attr_to_dict, load_model +from emf.common.config_parser import parse_app_properties +from emf.common.integrations import elastic, minio_api, rabbit +from emf.common.integrations.object_storage import models +from io import BytesIO +from zipfile import ZipFile, ZIP_DEFLATED +from emf.common.loadflow_tool.helper import get_model_outages + +logger = logging.getLogger(__name__) + +parse_app_properties(caller_globals=globals(), path=config.paths.model_quality.model_quality) + + +class HandlerModelQuality: + + def __init__(self): + self.minio_service = minio_api.ObjectStorage() + self.elastic_service = elastic.Elastic() + + def update_opdm_metadata_object(self, id: str, body: dict): + search_query = {"ids": {"values": [id]}} + response = self.elastic_service.client.search(index=f"{METADATA_ELK_INDEX}*", query=search_query, size=1) + index = response['hits']['hits'][0]['_index'] + self.elastic_service.update_document(index=index, id=id, body=body) + + def handle(self, message: bytes, properties: dict, **kwargs): + + logger.info(f"Loaded {message}") + + # Load OPDM metadata objects from binary to json + model_data = json.loads(message) + object_type = properties.headers['opde:Object-Type'] + + if object_type == 'CGM': + model_data['DATA'] = self.minio_service.download_object(model_data.get('minio-bucket'), + model_data.get('content_reference')) + logger.info(f"Loading merged model: {model_data['name']}") + network = load_cgm(model_data['DATA'], None) + + elif object_type == 'IGM': + model_data = models.get_content(metadata=model_data['opdm_object']) + latest_boundary = models.get_latest_boundary() + logger.info(f"Loading model...") + network = load_model(opdm_objects=[model_data, latest_boundary]) + else: + raise TypeError("Incorrect or missing metadata") + + if network: + qa_report = generate_quality_report(network, object_type, model_data) + else: + raise TypeError("Model was not loaded correctly, either missing in MinIO or incorrect data") + + # Send validation report to Elastic + if qa_report: + try: + response = self.elastic_service.send_to_elastic(index=ELK_QUALITY_INDEX, json_message=qa_report) + except Exception as error: + logger.error(f"Validation report sending to Elastic failed: {error}") + + logger.info(f"Queality report sent to elastic index {ELK_QUALITY_INDEX}") + else: + raise TypeError("Quality report generator failed, data not sent") + + return message, properties + + +def generate_quality_report(network, object_type, model_data): + + report = {} + + if object_type == "CGM" and model_data['merge_type'] == 'BA': + + # Check Kruonis generators + generators = network.get_generators(all_attributes=True) + kruonis_generators = generators[generators['name'].str.contains('KHE_G')] + if kruonis_generators: + gen_count = kruonis_generators['connected'].sum() + flag = gen_count > 2 + report.update({"kruonis_generators": gen_count, "kruonis_check": flag}) + else: + report.update({"kruonis_generators": None, "kruonis_check": False}) + + + # Check LT-PL crossborder flow + # TODO double check correct limit value + BORDER_LIMIT = 250 + d_lines = network.get_dangling_lines(all_attributes=True) + LT_PL_lines = d_lines[d_lines['name'].str.contains('Alytus-Elk')] + if LT_PL_lines: + flow_sum = LT_PL_lines['p'].sum() + flag = flow_sum < BORDER_LIMIT + report.update({"lt_pl_flow": flow_sum, "lt_pl_xborder_check": flag}) + else: + report.update({"lt_pl_flow": None, "lt_pl_xborder_check": False}) + + # Check cross-border line inconsistencies + # TODO log all line info + pairing_keys = d_lines.groupby('pairing_key')['connected'].nunique() + mismatch = len(pairing_keys[pairing_keys > 1].index.tolist()) + flag = mismatch < 1 + report.update({"xb_mismatch": mismatch, "xb_consitency_check": flag}) + + # Check model outage mismatch with outage plan + model_outages = pd.DataFrame(get_model_outages(network=network)) + + elif object_type == "IGM": + report = report + + return report + + +def load_cgm(network, parameters): + + if bytes == type(network): + try: + network = unzip_zipped_profiles(BytesIO(network)) + except Exception as error: + logger.error(error) + network = pp.network.load_from_binary_buffer(network, parameters=parameters) + if str == type(network): + try: + network = unzip_zipped_profiles(BytesIO(network)) + network = pp.network.load_from_binary_buffer(network, parameters=parameters) + except Exception as error: + logger.error(error) + network = pp.network.load(network, parameters=parameters) + + return network + + +def unzip_zipped_profiles(path_or_buffer: str): + """ + Extracts zipped profiles of a model. + Detects automatically if the zipped folder is read from file bath or BytesIO buffer. + :param path_or_buffer: path of zipped model or BytesIO buffer + :return: output_zip_buffer + """ + # Checks if path_or_buffer is string + if isinstance(path_or_buffer, str): + with open(path_or_buffer, 'rb') as original_zip_file: + original_zip_buffer = BytesIO(original_zip_file.read()) + + # Checks if path_or_buffer is BytesIO class + elif isinstance(path_or_buffer, BytesIO): + original_zip_buffer = path_or_buffer + else: + logger.error(f"Provided variable is nor string nor BytesIO object") + + output_zip_buffer = BytesIO() + + # Read the original zipped folder from provided buffer + with ZipFile(original_zip_buffer, 'r') as original_zip: + # Create a new zip file where we will store the unzipped XMLs + with ZipFile(output_zip_buffer, 'w', ZIP_DEFLATED) as new_zip: + # Iterate trough each file in the original zip + for file_name in original_zip.namelist(): + # Read the zipped xml file + with original_zip.open(file_name) as zipped_xml_file: + with ZipFile(zipped_xml_file) as zipped_xml: + # Extract each file and add it to the new zip + for xml_file_name in zipped_xml.namelist(): + xml_data = zipped_xml.read(xml_file_name) + # Add unzipped XML file to the new zip + new_zip.writestr(xml_file_name, xml_data) + + output_zip_buffer.seek(0) + + return output_zip_buffer + +def query_elk_uap(index, time_horizon=None): + + from datetime import datetime + + logger.info(f"Retrieving outages from ELK index: '{index}'") + + # now represents the time of the run, in P0W case it should be current time + now = datetime.now() + now = now.strftime("%Y-%m-%dT%H:%M") + "Z" + + if time_horizon == 'WK': + merge_type_list = ['week'] + elif time_horizon == 'MO': + merge_type_list = ['week', 'month'] + elif time_horizon == 'YR': + merge_type_list = ['year'] + + query = { + "bool": { + "must": [ + {"exists": {"field": "name"}}, + {"terms": {"Merge": merge_type_list}}, + ], + "filter": [{"range": {"reportParsedDate": {"lte": now, "gte": "now-2w"}}}], + } + } + response = get_docs_by_query(index=index, query=query, size=10000, return_df=True) + result = pd.DataFrame() + + if not response.empty: + + # Get only latest report data + response['reportParsedDate'] = pd.to_datetime(response['reportParsedDate']) + response = response[response['reportParsedDate'] == response['reportParsedDate'].max()] + # Only keep latest outages + duplicated_outages = response[response.duplicated('eic', keep=False)] + latest_duplicate = duplicated_outages.groupby('eic')['date_of_last_change'].idxmax() + response = response.loc[response.index.isin(latest_duplicate) | ~response['eic'].duplicated(keep=False)] + + response = response[response['outage_type'].isin(['OUT'])] + + response = response.sort_values(by=['eic', 'start_date', 'end_date']).reset_index(drop=True) + last_end_time = {} + + # Remove outage duplicate if there is time overlap + for _, row in response.iterrows(): + eic = row['eic'] + start_time = row['start_date'] + end_time = row['end_date'] + + if eic not in last_end_time or start_time > last_end_time[eic]: + result = pd.concat([result, pd.DataFrame([row])], ignore_index=True) + last_end_time[eic] = end_time + + return result diff --git a/emf/model_quality/quality_functions.py b/emf/model_quality/quality_functions.py new file mode 100644 index 00000000..e69de29b diff --git a/emf/model_quality/worker.py b/emf/model_quality/worker.py new file mode 100644 index 00000000..9bffe5d1 --- /dev/null +++ b/emf/model_quality/worker.py @@ -0,0 +1,20 @@ +import config +import logging +from emf.common.integrations import rabbit +from emf.common.config_parser import parse_app_properties +from emf.model_quality.model_quality import HandlerModelQuality + +# Initialize custom logger +logger = logging.getLogger(__name__) + +parse_app_properties(caller_globals=globals(), path=config.paths.model_quality.model_quality) + +# RabbitMQ consumer implementation +consumer = rabbit.RMQConsumer(queue=INPUT_RMQ_QUEUE, + message_handlers=[HandlerModelQuality()], + ) + +try: + consumer.run() +except KeyboardInterrupt: + consumer.stop() diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index 4d6eafad..f12c7a3f 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -8,7 +8,7 @@ import uuid from emf.common.loadflow_tool.helper import attr_to_dict, load_model from emf.common.config_parser import parse_app_properties -from emf.common.integrations import elastic, minio_api +from emf.common.integrations import elastic, minio_api, rabbit from emf.common.integrations.object_storage import models from emf.common.loadflow_tool.helper import load_opdm_data, clean_data_from_opdm_objects from emf.common.loadflow_tool import loadflow_settings @@ -141,10 +141,10 @@ def handle(self, message: bytes, properties: dict, **kwargs): start_time = time.time() # Load OPDM metadata objects from binary to json - opdm_objects = json.loads(message) + opdm_metadata = json.loads(message) # Get network models data from object storage - opdm_objects = [models.get_content(metadata=opdm_object) for opdm_object in opdm_objects] + opdm_objects = [models.get_content(metadata=opdm_object) for opdm_object in opdm_metadata] # Get the latest boundary set for validation latest_boundary = models.get_latest_boundary() @@ -176,6 +176,7 @@ def handle(self, message: bytes, properties: dict, **kwargs): report['content_reference'] = opdm_object['pmd:content-reference'] report['tso'] = opdm_object['pmd:TSO'] report['duration_s'] = round(time.time() - start_time, 3) + report['minio-bucket'] = opdm_object['minio-bucket'] except Exception as error: logger.error(f"Models validator failed with exception: {error}", exc_info=True) @@ -216,6 +217,14 @@ def handle(self, message: bytes, properties: dict, **kwargs): except Exception as error: logger.error(f"Validation report sending to Elastic failed: {error}") + # Send metadata to quality module queue + try: + report['opdm_object'] = opdm_object + rabbit.BlockingClient().publish(payload=json.dumps(report), exchange_name=RABBIT_QUALITY_EXCHANGE, + headers={"opde:Object-Type": "IGM"}) + except: + logger.error(f"Sending metadata to RabbitMQ failed: {error}") + logger.info(f"Model validation status: {valid} [duration {report['duration_s']}s]") return message, properties @@ -234,9 +243,15 @@ def handle(self, message: bytes, properties: dict, **kwargs): #logging.getLogger('powsybl').setLevel(1) opdm = OPDM() - latest_boundary = opdm.get_latest_boundary() + + opdm_metadata = json.loads(message) + + # Get network models data from object storage + opdm_objects = [models.get_content(metadata=opdm_object) for opdm_object in opdm_metadata] + + # latest_boundary = opdm.get_latest_boundary() available_models = opdm.get_latest_models_and_download(time_horizon='1D', - scenario_date="2025-04-28T09:30", + scenario_date="2025-05-27T09:30", tso="AST") validated_models = [] From 5b40385f6e8d028ae2e303457eb118daca83fd49 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Thu, 5 Jun 2025 15:56:45 +0300 Subject: [PATCH 04/67] ci: manual trigger posibility --- .github/workflows/publish-workers.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-workers.yml b/.github/workflows/publish-workers.yml index ea2a7871..7abca0fc 100644 --- a/.github/workflows/publish-workers.yml +++ b/.github/workflows/publish-workers.yml @@ -7,6 +7,7 @@ on: - "v*" branches: - main + workflow_dispatch: jobs: build_workers: From 5f5ff286a15b8dc8e86fe2633089c7c2cc0f8504 Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Sun, 8 Jun 2025 16:23:38 +0300 Subject: [PATCH 05/67] Merged model OPDM object send to ELK --- config/cgm_worker/merger.properties | 1 + emf/model_merger/merge_functions.py | 14 ++++++++++++++ emf/model_merger/model_merger.py | 15 +++++++-------- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/config/cgm_worker/merger.properties b/config/cgm_worker/merger.properties index 9b5fd173..5c940c47 100644 --- a/config/cgm_worker/merger.properties +++ b/config/cgm_worker/merger.properties @@ -5,6 +5,7 @@ INPUT_MINIO_FOLDER = IGM OUTPUT_MINIO_BUCKET = opde-confidential-models OUTPUT_MINIO_FOLDER = EMFOS/RMM MERGE_REPORT_ELK_INDEX = emfos-merge-reports +OPDE_MODELS_ELK_INDEX = emfos-opde-models RABBIT_QUALITY_EXCHANGE = emf-quality.test SMALL_ISLAND_SIZE = 10 MERGE_LOAD_FLOW_SETTINGS = CGM_RELAXED_3_CUSTOM diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index a1a4c41f..3fa69e61 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -101,6 +101,20 @@ def create_opdm_object_meta(object_id, return opdm_object_meta +def update_cgm_opdm_object_meta(opdm_object_meta, merged_model): + + opdm_object_meta.update({'opde:Object-Type': 'CGM', + "pmd:isFullModel": "true", + "pmd:Area": opdm_object_meta['pmd:mergingArea'], + "pmd:version": opdm_object_meta['pmd:versionNumber'], + "pmd:modelid": opdm_object_meta['pmd:fullModel_ID'], + 'pmd:content-reference': merged_model.content_reference, + "pmd:modelPartReference": opdm_object_meta['pmd:mergingEntity']+'-'+opdm_object_meta['pmd:mergingArea'], + }) + + return opdm_object_meta + + def update_FullModel_from_OpdmObject(data, opdm_object): return triplets.cgmes_tools.update_FullModel_from_dict(data, metadata={ "Model.version": f"{int(opdm_object['pmd:versionNumber']):03d}", diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index d7ecf511..30c6a2c7 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -20,7 +20,7 @@ from emf.common.loadflow_tool.helper import opdmprofile_to_bytes, attr_to_dict from emf.model_merger import merge_functions from emf.model_merger import scaler -from emf.model_merger.replacement import run_replacement, get_available_tsos, run_replacement_local +from emf.model_merger.replacement import run_replacement, get_available_tsos from emf.task_generator.task_generator import update_task_status from emf.common.logging.custom_logger import get_elk_logging_handler # TODO - move this async solution to some common module @@ -333,10 +333,6 @@ def handle(self, task_object: dict, properties: dict, **kwargs): enable_temp_fixes=post_temp_fixes, time_horizon=new_time_horizon) - opdm_object_meta['pmd:content-reference'] = '' - opdm_object_meta['opde:Object-Type'] = '' - - # Package both input models and exported CGM profiles to in memory zip files serialized_data = merge_functions.export_to_cgmes_zip([ssh_data, sv_data]) post_p_end = datetime.datetime.now(datetime.UTC) @@ -407,14 +403,17 @@ def handle(self, task_object: dict, properties: dict, **kwargs): merged_model.duration_s = (end_time - merge_start).total_seconds() merged_model.content_reference = merged_model_object.name - # Send merge report to Elastic + # Update OPDM object data with CGM relevant data + opdm_object_meta = merge_functions.update_cgm_opdm_object_meta(opdm_object_meta, merged_model) + + # Send merge report and opdm object metadata to Elastic if model_merge_report_send_to_elk: logger.info(f"Sending merge report to Elastic") try: merge_report = merge_functions.generate_merge_report(merged_model=merged_model, task=task) try: - response = elastic.Elastic.send_to_elastic(index=MERGE_REPORT_ELK_INDEX, - json_message=merge_report) + response = elastic.Elastic.send_to_elastic(index=MERGE_REPORT_ELK_INDEX, json_message=merge_report) + response2 = elastic.Elastic.send_to_elastic(index=OPDE_MODELS_ELK_INDEX, json_message=opdm_object_meta) except Exception as error: logger.error(f"Merge report sending to Elastic failed: {error}") except Exception as error: From 8995fd4d1cfe8baa05be05a6dcac6aa335969456 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 9 Jun 2025 11:47:35 +0300 Subject: [PATCH 06/67] refactor: task generator elk index configured in one place refactor: removed dependency from object storage --- config/integrations/object_storage.properties | 1 - .../task_generator/task_generator.properties | 2 +- .../integrations/object_storage/tasks.py | 20 ------ emf/task_generator/task_generator.py | 71 ++++++++++--------- emf/task_generator/worker.py | 10 +-- 5 files changed, 42 insertions(+), 62 deletions(-) delete mode 100644 emf/common/integrations/object_storage/tasks.py diff --git a/config/integrations/object_storage.properties b/config/integrations/object_storage.properties index 5fc8d47e..7086de81 100644 --- a/config/integrations/object_storage.properties +++ b/config/integrations/object_storage.properties @@ -1,4 +1,3 @@ [MAIN] -ELASTIC_TASKS_INDEX = emfos-tasks ELASTIC_MODELS_INDEX = emfos-opde-models ELASTIC_SCHEDULES_INDEX = emfos-schedules \ No newline at end of file diff --git a/config/task_generator/task_generator.properties b/config/task_generator/task_generator.properties index b3a78214..b2e4754a 100644 --- a/config/task_generator/task_generator.properties +++ b/config/task_generator/task_generator.properties @@ -1,6 +1,6 @@ [MAIN] RMQ_EXCHANGE = emf-tasks -TASK_ELK_INDEX = +TASK_ELK_INDEX = emfos-tasks TASK_WINDOW_DURATION = PT1M TASK_WINDOW_REFERENCE = currentMinuteStart TIMESTAMP = diff --git a/emf/common/integrations/object_storage/tasks.py b/emf/common/integrations/object_storage/tasks.py deleted file mode 100644 index bfb64993..00000000 --- a/emf/common/integrations/object_storage/tasks.py +++ /dev/null @@ -1,20 +0,0 @@ -from emf.common.integrations.object_storage import * -from emf.common.integrations.object_storage.models import query_data -from functools import partial -import logging -logger = logging.getLogger(__name__) - - -def publish_tasks(tasks: list, index=ELASTIC_TASKS_INDEX): - - for task in tasks: - logger.info(f"Adding task to {ELASTIC_TASKS_INDEX}") - elastic_service.send_to_elastic( - index=index, - json_message=task, - id=task.get("@id") - ) - - -query_tasks = partial(query_data, index=ELASTIC_TASKS_INDEX) - diff --git a/emf/task_generator/task_generator.py b/emf/task_generator/task_generator.py index 6f2fd133..d04d0a52 100644 --- a/emf/task_generator/task_generator.py +++ b/emf/task_generator/task_generator.py @@ -1,17 +1,14 @@ import aniso8601 from datetime import datetime - -from emf.common.time_helper import parse_duration, convert_to_utc, convert_to_timezone, timezone, reference_times, utcnow -from emf.common.integrations.object_storage.tasks import publish_tasks -from emf.common.integrations.object_storage.models import query_data -from emf.common.config_parser import parse_app_properties import config from uuid import uuid4 import croniter -from pathlib import Path import json import logging from os import getlogin +from emf.common.time_helper import parse_duration, convert_to_utc, convert_to_timezone, timezone, reference_times, utcnow +from emf.common.integrations.elastic import Elastic +from emf.common.config_parser import parse_app_properties logger = logging.getLogger(__name__) parse_app_properties(globals(), config.paths.task_generator.task_generator) @@ -51,7 +48,6 @@ def generate_tasks(task_window_duration: str, time_frames = {time_frame["@id"].split("/")[-1]: time_frame for time_frame in timeframe_conf} # Load the process configuration from the specified file. - for process in process_conf: # Loop through each run in the process configuration. @@ -65,7 +61,7 @@ def generate_tasks(task_window_duration: str, if timetravel_now: now = aniso8601.parse_datetime(timetravel_now) - logger.info(f"Now -> {now}") + logger.info(f"Now: {now}") # Calculate the start and end of the task window for the run. run_window_start = reference_times[task_window_reference](now) @@ -84,20 +80,20 @@ def generate_tasks(task_window_duration: str, else: _ = runs.get_next(datetime) - logger.info(f"Next run of {run['@id']} at {run_timestamp}") + logger.info(f"Next run of {run['@id']} at: {run_timestamp}") - if not (run_timestamp >= run_window_start and run_timestamp <= run_window_end): - logger.info(f"Run at {run_timestamp} not in window [{run_window_start}/{run_window_end}] -> {run['@id']}") + if not (run_window_start <= run_timestamp <= run_window_end): + logger.info(f"Run at {run_timestamp} not in window [{run_window_start}/{run_window_end}]: {run['@id']}") # Loop through each timestamp in the current run. while run_timestamp <= run_window_end: - logger.info(f"Run at {run_timestamp} in window [{run_window_start}/{run_window_end}] -> {run['@id']}") + logger.info(f"Run at {run_timestamp} in window [{run_window_start}/{run_window_end}]: {run['@id']}") # Get the reference time for the current timestamp in the time frame. reference_time_start = reference_times[time_frame["reference_time_start"]](run_timestamp) reference_time_end = reference_times[time_frame["reference_time_end"]](run_timestamp) - # Change reference time according to time shift config + # Change reference time according to time shift config. if process_time_shift: reference_time_start = reference_time_start + parse_duration(process_time_shift) reference_time_end = reference_time_end + parse_duration(process_time_shift) @@ -110,7 +106,7 @@ def generate_tasks(task_window_duration: str, job_period_start_utc = convert_to_utc(job_period_start) job_period_end_utc = convert_to_utc(job_period_end) - # Calculate the open and close times for the gate (the window of time before the job period start time during wich the job/tasks must be done). + # Calculate the open and close times for the gate (the window of time before the job period start time during which the job/tasks must be done). gate_open_utc = job_period_start_utc - parse_duration(run["gate_open"]) gate_close_utc = job_period_start_utc - parse_duration(run["gate_close"]) @@ -133,7 +129,7 @@ def generate_tasks(task_window_duration: str, task_id = str(uuid4()) task_timestamp = utcnow().isoformat() - logger.info(f"Task {timestamp_utc} in window [{job_period_start_utc}/{job_period_end_utc}] -> Job: {job_id}") + logger.info(f"Task {timestamp_utc} in window [{job_period_start_utc}/{job_period_end_utc}] for job: {job_id}") task = { "@context": "https://example.com/task_context.jsonld", @@ -174,13 +170,10 @@ def generate_tasks(task_window_duration: str, task["task_tags"].extend(run.get("tags", [])) # Check if task already exists, then set version number accordingly - if TASK_ELK_INDEX: - set_task_version(task, TASK_ELK_INDEX) - else: - set_task_version(task) + set_task_version(task=task) # Update task status - update_task_status(task, "created") + update_task_status(task=task, status_text="created") # Return Task yield task @@ -237,13 +230,13 @@ def filter_and_flatten_dict(nested_dict: dict, keys: list): return {key: flattened[key] for key in keys if key in flattened} -def update_task_status(task, status_text, publish=True): +def update_task_status(task: dict, status_text: str, publish: bool = True): """Update task status Will update task_update_time Will update task_status Will append new status to task_status_trace""" - logger.info(f"Updating Task status to -> {status_text}") + logger.info(f"Updating Task status to: {status_text}") utc_now = datetime.utcnow().isoformat() @@ -255,21 +248,28 @@ def update_task_status(task, status_text, publish=True): "timestamp": utc_now }) - # TODO - better handling if elk is not available, possibly set elk connection timout really small or refactro the sending to happen via rabbit + # TODO - better handling if elk is not available, possibly set elk connection timeout really small or refactor the sending to happen via rabbit if publish: try: - publish_tasks([task]) - except: - logger.warning("Task publication to Elastic failed") - - -def set_task_version(task, elk_index='emfos-tasks*'): - query = {'task_properties.timestamp_utc': task['task_properties']['timestamp_utc'], - 'task_properties.time_horizon': task['task_properties']['time_horizon'], - 'task_properties.merge_type': task['task_properties']['merge_type']} - + for task in tasks: + Elastic.send_to_elastic( + index=TASK_ELK_INDEX, + json_message=task, + id=task.get("@id") + ) + except Exception as e: + logger.warning(f"Task publication to Elastic failed with error: {e}") + + +def set_task_version(task: dict, elk_index: str = TASK_ELK_INDEX): + query = { + 'task_properties.timestamp_utc': task['task_properties']['timestamp_utc'], + 'task_properties.time_horizon': task['task_properties']['time_horizon'], + 'task_properties.merge_type': task['task_properties']['merge_type'], + } + service = Elastic() try: - task_list = query_data(query, index=elk_index) + task_list = service.get_docs_by_query(index=TASK_ELK_INDEX, query=query) if task_list: latest_version = max(int(item['task_properties'].get('version', "0")) for item in task_list if item['task_properties'].get('version', 1)) if task['task_properties']['version'] == 'AUTO': @@ -282,8 +282,9 @@ def set_task_version(task, elk_index='emfos-tasks*'): if task['task_properties']['version'] == 'AUTO': task['task_properties']['version'] = '001' - except: + except Exception as e: logger.warning("Elastic query for task versioning unsuccessful, version not updated") + logger.warning(f"Exception traceback: {e}") if __name__ == "__main__": diff --git a/emf/task_generator/worker.py b/emf/task_generator/worker.py index c74485ac..72510e5b 100644 --- a/emf/task_generator/worker.py +++ b/emf/task_generator/worker.py @@ -23,13 +23,13 @@ if tasks: logger.info(f"Creating connection to RMQ") rabbit_service = rabbit.BlockingClient() - logger.info(f"Sending tasks to Rabbit exchange '{RMQ_EXCHANGE}'") + logger.info(f"Sending tasks to Rabbit exchange: {RMQ_EXCHANGE}") for task in tasks: elk_handler.start_trace(task) - rabbit_service.publish(payload=json.dumps(task), exchange_name=RMQ_EXCHANGE, headers=filter_and_flatten_dict(task, TASK_HEADER_KEYS.split(","))) + rabbit_service.publish(payload=json.dumps(task), + exchange_name=RMQ_EXCHANGE, + headers=filter_and_flatten_dict(task, TASK_HEADER_KEYS.split(","))) elk_handler.stop_trace() else: - logger.info("No tasks generated at current time.") - - + logger.info("No tasks generated at current timeframe, exiting worker.") From b2ec1231f7227375949d1ce5ba4d1da2c15bc985 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 9 Jun 2025 15:00:35 +0300 Subject: [PATCH 07/67] bugfix: task generator improved with more logging and handling automatic versioning --- emf/common/integrations/elastic.py | 4 ++ emf/task_generator/task_generator.py | 62 ++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 18 deletions(-) diff --git a/emf/common/integrations/elastic.py b/emf/common/integrations/elastic.py index 1238d2b3..adf7a658 100644 --- a/emf/common/integrations/elastic.py +++ b/emf/common/integrations/elastic.py @@ -149,6 +149,10 @@ def update_document(self, index: str, id: str, body: dict): def get_docs_by_query(self, index: str, query: dict, size: int | None = None, return_df: bool = True): + # Validate index definition to be able to search all index by pattern + if "*" not in index: + index = f"{index}*" + response = self.client.search(index=index, query=query, size=size) if self.debug: logger.info(f"Returned total {response['hits']['total']['value']} document") diff --git a/emf/task_generator/task_generator.py b/emf/task_generator/task_generator.py index d04d0a52..642dffef 100644 --- a/emf/task_generator/task_generator.py +++ b/emf/task_generator/task_generator.py @@ -261,26 +261,49 @@ def update_task_status(task: dict, status_text: str, publish: bool = True): logger.warning(f"Task publication to Elastic failed with error: {e}") -def set_task_version(task: dict, elk_index: str = TASK_ELK_INDEX): +def set_task_version(task: dict): + + # Check versioning mode + auto_versioning_enabled = False + if task['task_properties']['version'] == 'AUTO': + logger.debug("Task versioning set to AUTO mode") + auto_versioning_enabled = True + query = { - 'task_properties.timestamp_utc': task['task_properties']['timestamp_utc'], - 'task_properties.time_horizon': task['task_properties']['time_horizon'], - 'task_properties.merge_type': task['task_properties']['merge_type'], + "bool": { + "must": [ + {"match": {"task_properties.timestamp_utc": task['task_properties']['timestamp_utc']}}, + {"term": {"task_properties.time_horizon.keyword": task['task_properties']['time_horizon']}}, + {"term": {"task_properties.merge_type.keyword": task['task_properties']['merge_type']}}, + ] + } } + service = Elastic() try: - task_list = service.get_docs_by_query(index=TASK_ELK_INDEX, query=query) - if task_list: - latest_version = max(int(item['task_properties'].get('version', "0")) for item in task_list if item['task_properties'].get('version', 1)) - if task['task_properties']['version'] == 'AUTO': - task['task_properties']['version'] = str(int(latest_version) + 1).zfill(3) - elif int(latest_version) > int(task['task_properties']['version']): - task['task_properties']['version'] = str(int(latest_version) + 1).zfill(3) - else: - logger.error("Version set lower than existing model") + updated_version = None + tasks_df = service.get_docs_by_query(index=TASK_ELK_INDEX, query=query) + + if tasks_df.empty: + logger.info(f"No previous runs found for task, using version from configuration: {task['task_properties']['version']}") + if auto_versioning_enabled: + logger.info("Task versioning mode 'AUTO', defaulting to: '001'") + updated_version = '001' else: - if task['task_properties']['version'] == 'AUTO': - task['task_properties']['version'] = '001' + latest_version = tasks_df['task_properties.version'].max() + logger.info(f"Latest available task version: {latest_version}") + if auto_versioning_enabled: + updated_version = str(int(latest_version) + 1).zfill(3) + elif int(latest_version) >= int(task['task_properties']['version']): + logger.warning("Latest available version is equal or higher than defined in task, increasing from latest") + updated_version = str(int(latest_version) + 1).zfill(3) + else: + logger.info("Using version for task configuration") + updated_version = task['task_properties']['version'] + + if updated_version: + task['task_properties']['version'] = updated_version + logger.info(f"Version set to: {updated_version}") except Exception as e: logger.warning("Elastic query for task versioning unsuccessful, version not updated") @@ -298,10 +321,13 @@ def set_task_version(task: dict, elk_index: str = TASK_ELK_INDEX): task_window_duration = "P1D" task_window_reference = "currentDayStart" - timeframe_conf = "../../config/task_generator/timeframe_conf.json" - process_conf = "../../config/task_generator/process_conf.json" + timeframe_conf = config.paths.task_generator.timeframe_conf + process_conf = config.paths.task_generator.process_conf + + timeframe_config_json = json.load(timeframe_conf) + process_config_json = json.load(process_conf) - tasks = list(generate_tasks(task_window_duration, task_window_reference, process_conf, timeframe_conf)) + tasks = list(generate_tasks(task_window_duration, task_window_reference, process_config_json, timeframe_config_json)) tasks_table = pandas.json_normalize(tasks) print(tasks_table["process_id"].value_counts()) From 0946a953bf926fae197c6a71384819970b96ad8b Mon Sep 17 00:00:00 2001 From: MATISS <136691671+MK1295@users.noreply.github.com> Date: Mon, 9 Jun 2025 15:45:54 +0300 Subject: [PATCH 08/67] Update process_conf.json --- config/task_generator/process_conf.json | 40 +++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/config/task_generator/process_conf.json b/config/task_generator/process_conf.json index 386f8132..c9d3d8ab 100644 --- a/config/task_generator/process_conf.json +++ b/config/task_generator/process_conf.json @@ -312,6 +312,46 @@ "send_merge_report": "True" } }, + { + "@context": "https://example.com/schemas/run.jsonld", + "@id": "https://example.com/runs/IntraDayRMM/4", + "@type": "Run", + "process_id": "https://example.com/processes/RMM_CREATION", + "valid_from": "", + "valid_to": "", + "gate_open": "PT1H", + "gate_close": "PT45M", + "run_at": "00 15 29 02 3", + "time_frame": "ID", + "data_timestamps": "30 * * * *", + "data_resolution": "PT1H", + "tags": [], + "properties": { + "merge_type": "BA", + "merging_entity": "BALTICRCC", + "included": [ + "AST", + "PSE", + "ELERING", + "LITGRID" + ], + "excluded": [], + "local_import": [], + "time_horizon": "ID", + "version": "001", + "mas": "http://www.baltic-rsc.eu/OperationalPlanning/RMM", + "replacement": "True", + "replacement_local":"False", + "pre_temp_fixes":"True", + "post_temp_fixes":"True", + "fix_net_interchange2": "False", + "force_outage_fix":"False", + "scaling": "False", + "upload_to_opdm": "False", + "upload_to_minio": "True", + "send_merge_report": "True" + } + }, { "@context": "https://example.com/schemas/run.jsonld", "@id": "https://example.com/runs/DayAheadRMM", From 26768e080b81bc5603590d9515b7f8fc559f52fb Mon Sep 17 00:00:00 2001 From: MATISS <136691671+MK1295@users.noreply.github.com> Date: Mon, 9 Jun 2025 15:47:21 +0300 Subject: [PATCH 09/67] Update process_conf.json --- config/task_generator/process_conf.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/task_generator/process_conf.json b/config/task_generator/process_conf.json index c9d3d8ab..03d53b86 100644 --- a/config/task_generator/process_conf.json +++ b/config/task_generator/process_conf.json @@ -333,7 +333,7 @@ "AST", "PSE", "ELERING", - "LITGRID" + "LITGRID" ], "excluded": [], "local_import": [], From 01e6f734b826084b8e39f0e170c867833089cf9f Mon Sep 17 00:00:00 2001 From: MATISS <136691671+MK1295@users.noreply.github.com> Date: Mon, 9 Jun 2025 15:56:31 +0300 Subject: [PATCH 10/67] Update process_conf.json --- config/task_generator/process_conf.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/task_generator/process_conf.json b/config/task_generator/process_conf.json index 03d53b86..1c267498 100644 --- a/config/task_generator/process_conf.json +++ b/config/task_generator/process_conf.json @@ -314,7 +314,7 @@ }, { "@context": "https://example.com/schemas/run.jsonld", - "@id": "https://example.com/runs/IntraDayRMM/4", + "@id": "https://example.com/runs/IntraDayRMM/EOD", "@type": "Run", "process_id": "https://example.com/processes/RMM_CREATION", "valid_from": "", From 3d4a2a948ed1a75610d6e2cc7fd4206af2c9e887 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 9 Jun 2025 16:03:24 +0300 Subject: [PATCH 11/67] bugfix: update task status fixed as it works without loop --- emf/task_generator/task_generator.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/emf/task_generator/task_generator.py b/emf/task_generator/task_generator.py index 642dffef..08002ec9 100644 --- a/emf/task_generator/task_generator.py +++ b/emf/task_generator/task_generator.py @@ -251,12 +251,7 @@ def update_task_status(task: dict, status_text: str, publish: bool = True): # TODO - better handling if elk is not available, possibly set elk connection timeout really small or refactor the sending to happen via rabbit if publish: try: - for task in tasks: - Elastic.send_to_elastic( - index=TASK_ELK_INDEX, - json_message=task, - id=task.get("@id") - ) + Elastic.send_to_elastic(index=TASK_ELK_INDEX, json_message=task, id=task.get("@id")) except Exception as e: logger.warning(f"Task publication to Elastic failed with error: {e}") From dfdcf5fbbd94067713ee68396823e7be9842cd0a Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 9 Jun 2025 16:04:57 +0300 Subject: [PATCH 12/67] logging: from warning to error --- emf/task_generator/task_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emf/task_generator/task_generator.py b/emf/task_generator/task_generator.py index 08002ec9..27fd9a40 100644 --- a/emf/task_generator/task_generator.py +++ b/emf/task_generator/task_generator.py @@ -253,7 +253,7 @@ def update_task_status(task: dict, status_text: str, publish: bool = True): try: Elastic.send_to_elastic(index=TASK_ELK_INDEX, json_message=task, id=task.get("@id")) except Exception as e: - logger.warning(f"Task publication to Elastic failed with error: {e}") + logger.error(f"Task publication to Elastic failed with error: {e}") def set_task_version(task: dict): From 3742109ff63f29f6d6dfce75adaa90467f01074e Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Wed, 11 Jun 2025 10:18:49 +0300 Subject: [PATCH 13/67] logging: added logging message for schedule query --- emf/model_merger/model_merger.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 30c6a2c7..c74b4d2b 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -296,6 +296,9 @@ def handle(self, task_object: dict, properties: dict, **kwargs): if not schedule_start: schedule_start = scenario_datetime + # Print out the schedule parameters for scaling + logger.info(f"Querying schedules for: {schedule_time_horizon} {schedule_start}") + # Get aligned schedules ac_schedules = query_acnp_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) dc_schedules = query_hvdc_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) From 713838c0b91dc43e220350b5c5e61edcfa2772e6 Mon Sep 17 00:00:00 2001 From: MATISS <136691671+MK1295@users.noreply.github.com> Date: Fri, 13 Jun 2025 10:07:39 +0300 Subject: [PATCH 14/67] Update scaler.properties --- config/cgm_worker/scaler.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/cgm_worker/scaler.properties b/config/cgm_worker/scaler.properties index 7225c735..12be5e26 100644 --- a/config/cgm_worker/scaler.properties +++ b/config/cgm_worker/scaler.properties @@ -1,4 +1,4 @@ [MAIN] -MAX_ITERATION = 10 +MAX_ITERATION = 15 BALANCE_THRESHOLD = 2 -DEBUG = True \ No newline at end of file +DEBUG = True From 1d6ca3a87e60748732503ddcbe5ceef1e0efe515 Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Fri, 13 Jun 2025 14:44:02 +0300 Subject: [PATCH 15/67] Model quality triplets refactoring --- config/model_quality/model_quality.properties | 3 +- emf/model_quality/model_quality.py | 187 ++++++++++++++---- emf/model_validator/model_statistics.py | 2 +- 3 files changed, 148 insertions(+), 44 deletions(-) diff --git a/config/model_quality/model_quality.properties b/config/model_quality/model_quality.properties index 263a766e..5dca1634 100644 --- a/config/model_quality/model_quality.properties +++ b/config/model_quality/model_quality.properties @@ -1,3 +1,4 @@ [MAIN] INPUT_RMQ_QUEUE = model-quality.test -ELK_QUALITY_INDEX = test-emf-model-quality \ No newline at end of file +ELK_QUALITY_INDEX = test-emf-model-quality +ELK_STATISTICS_INDEX = test-emf-model-statistics \ No newline at end of file diff --git a/emf/model_quality/model_quality.py b/emf/model_quality/model_quality.py index 7be4e2b3..efa23ff0 100644 --- a/emf/model_quality/model_quality.py +++ b/emf/model_quality/model_quality.py @@ -3,13 +3,14 @@ import config import json import pypowsybl as pp -from emf.common.loadflow_tool.helper import attr_to_dict, load_model from emf.common.config_parser import parse_app_properties from emf.common.integrations import elastic, minio_api, rabbit from emf.common.integrations.object_storage import models from io import BytesIO from zipfile import ZipFile, ZIP_DEFLATED from emf.common.loadflow_tool.helper import get_model_outages +from triplets.rdf_parser import load_all_to_dataframe +from emf.model_validator.model_statistics import get_system_metrics logger = logging.getLogger(__name__) @@ -22,39 +23,57 @@ def __init__(self): self.minio_service = minio_api.ObjectStorage() self.elastic_service = elastic.Elastic() - def update_opdm_metadata_object(self, id: str, body: dict): - search_query = {"ids": {"values": [id]}} - response = self.elastic_service.client.search(index=f"{METADATA_ELK_INDEX}*", query=search_query, size=1) - index = response['hits']['hits'][0]['_index'] - self.elastic_service.update_document(index=index, id=id, body=body) - def handle(self, message: bytes, properties: dict, **kwargs): logger.info(f"Loaded {message}") # Load OPDM metadata objects from binary to json - model_data = json.loads(message) + model_meta = json.loads(message) object_type = properties.headers['opde:Object-Type'] if object_type == 'CGM': - model_data['DATA'] = self.minio_service.download_object(model_data.get('minio-bucket'), - model_data.get('content_reference')) - logger.info(f"Loading merged model: {model_data['name']}") - network = load_cgm(model_data['DATA'], None) + model_data = self.minio_service.download_object(model_meta.get('minio-bucket'), + model_meta.get('content_reference')) + logger.info(f"Loading merged model: {model_meta['name']}") + unzipped = process_zipped_cgm(model_data) + network = load_all_to_dataframe(unzipped) + # TODO test if IGM retrieving works as intended elif object_type == 'IGM': - model_data = models.get_content(metadata=model_data['opdm_object']) - latest_boundary = models.get_latest_boundary() + model_meta = models.get_content(metadata=model_meta['opdm_object']) logger.info(f"Loading model...") - network = load_model(opdm_objects=[model_data, latest_boundary]) + try: + data = BytesIO(model_meta["opdm:Profile"]["DATA"]) + network = load_all_to_dataframe(data) + except: + logger.error("Failed to load IGM") + network = pd.DataFrame else: - raise TypeError("Incorrect or missing metadata") + logger.error("Incorrect or missing metadata") + network = pd.DataFrame - if network: - qa_report = generate_quality_report(network, object_type, model_data) + if not network.empty: + qa_report = generate_quality_report(network, object_type, model_meta) + try: + # TODO move statistics function to quality functions file or move statistics file to quality directory + model_statistics = get_system_metrics(network) + except: + logger.error("Failed to get model statistics") else: raise TypeError("Model was not loaded correctly, either missing in MinIO or incorrect data") + if model_statistics: + model_statistics.update({k: v for k, v in model_meta.items() if k.startswith('@')}) + model_statistics.update(properties.headers) + try: + response = self.elastic_service.send_to_elastic(index=ELK_STATISTICS_INDEX, json_message=model_statistics) + except Exception as error: + logger.error(f"Statistics report sending to Elastic failed: {error}") + + logger.info(f"Statistics report sent to elastic index {ELK_STATISTICS_INDEX}") + else: + raise TypeError("Statistics report generator failed, data not sent") + # Send validation report to Elastic if qa_report: try: @@ -69,45 +88,89 @@ def handle(self, message: bytes, properties: dict, **kwargs): return message, properties -def generate_quality_report(network, object_type, model_data): +def generate_quality_report(network, object_type, model_meta): report = {} - if object_type == "CGM" and model_data['merge_type'] == 'BA': + if object_type == "CGM" and model_meta['merge_type'] == 'BA': # Check Kruonis generators - generators = network.get_generators(all_attributes=True) - kruonis_generators = generators[generators['name'].str.contains('KHE_G')] - if kruonis_generators: - gen_count = kruonis_generators['connected'].sum() - flag = gen_count > 2 + generators = network.type_tableview('SynchronousMachine').rename_axis('Terminal').reset_index() + kruonis_generators = generators[generators['IdentifiedObject.name'].str.contains('KHAE_G')] + + if not kruonis_generators.empty: + gen_count = kruonis_generators[kruonis_generators['RotatingMachine.p'] > 0].shape[0] + flag = gen_count < 3 report.update({"kruonis_generators": gen_count, "kruonis_check": flag}) else: report.update({"kruonis_generators": None, "kruonis_check": False}) - # Check LT-PL crossborder flow - # TODO double check correct limit value - BORDER_LIMIT = 250 - d_lines = network.get_dangling_lines(all_attributes=True) - LT_PL_lines = d_lines[d_lines['name'].str.contains('Alytus-Elk')] - if LT_PL_lines: - flow_sum = LT_PL_lines['p'].sum() - flag = flow_sum < BORDER_LIMIT - report.update({"lt_pl_flow": flow_sum, "lt_pl_xborder_check": flag}) - else: - report.update({"lt_pl_flow": None, "lt_pl_xborder_check": False}) + # Check LT-PL crossborder flow + try: + control_areas = (network.type_tableview('ControlArea') + .rename_axis('ControlArea') + .reset_index())[['ControlArea', 'ControlArea.netInterchange', 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', 'IdentifiedObject.name']] + except KeyError: + control_areas = network.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + ssh_areas = network.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + control_areas = control_areas.merge(ssh_areas, on='ControlArea')[ + ['ControlArea', 'ControlArea.netInterchange', + 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', + 'IdentifiedObject.name']] + tie_flows = (network.type_tableview('TieFlow') + .rename_axis('TieFlow').rename(columns={'TieFlow.ControlArea': 'ControlArea', + 'TieFlow.Terminal': 'Terminal'}) + .reset_index())[['ControlArea', 'Terminal', 'TieFlow.positiveFlowIn']] + tie_flows = tie_flows.merge(control_areas[['ControlArea']], on='ControlArea') + try: + terminals = (network.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal', 'ACDCTerminal.connected']] + except KeyError: + terminals = (network.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal']] + tie_flows = tie_flows.merge(terminals, on='Terminal') + try: + power_flows_pre = (network.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + tie_flows = tie_flows.merge(power_flows_pre, on='Terminal', how='left') + except Exception: + logger.error(f"Was not able to get tie flows from original models") + power_flows_post = (network.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + + tie_flows = tie_flows.merge(power_flows_post, on='Terminal', how='left', + suffixes=('_pre', '_post')) + # TODO double check correct limit value + # BORDER_LIMIT = 250 + # d_lines = network.get_dangling_lines(all_attributes=True) + # LT_PL_lines = d_lines[d_lines['name'].str.contains('Alytus-Elk')] + # if LT_PL_lines: + # flow_sum = LT_PL_lines['p'].sum() + # flag = flow_sum < BORDER_LIMIT + # report.update({"lt_pl_flow": flow_sum, "lt_pl_xborder_check": flag}) + # else: + + # TODO fix border flow + report.update({"lt_pl_flow": None, "lt_pl_xborder_check": False}) + + # TODO remake into Triplets # Check cross-border line inconsistencies # TODO log all line info - pairing_keys = d_lines.groupby('pairing_key')['connected'].nunique() - mismatch = len(pairing_keys[pairing_keys > 1].index.tolist()) - flag = mismatch < 1 - report.update({"xb_mismatch": mismatch, "xb_consitency_check": flag}) + # pairing_keys = d_lines.groupby('pairing_key')['connected'].nunique() + # mismatch = len(pairing_keys[pairing_keys > 1].index.tolist()) + # flag = mismatch < 1 + # report.update({"xb_mismatch": mismatch, "xb_consitency_check": flag}) - # Check model outage mismatch with outage plan - model_outages = pd.DataFrame(get_model_outages(network=network)) + # TODO Check model outage mismatch with outage plan + # model_outages = pd.DataFrame(get_model_outages(network=network)) + # TODO define IGM quality rules elif object_type == "IGM": report = report @@ -133,6 +196,22 @@ def load_cgm(network, parameters): return network +def process_zipped_cgm(zipped_bytes, processed=[]): + + with ZipFile(BytesIO(zipped_bytes)) as zf: + for name in zf.namelist(): + with zf.open(name) as file: + content = file.read() + if name.endswith('.zip'): + process_zipped_cgm(content) + elif name.endswith('.xml'): + file_object = BytesIO(content) + file_object.name = name + processed.append(file_object) + + return processed + + def unzip_zipped_profiles(path_or_buffer: str): """ Extracts zipped profiles of a model. @@ -169,9 +248,11 @@ def unzip_zipped_profiles(path_or_buffer: str): new_zip.writestr(xml_file_name, xml_data) output_zip_buffer.seek(0) + output_zip_buffer.name = 'model' return output_zip_buffer + def query_elk_uap(index, time_horizon=None): from datetime import datetime @@ -227,3 +308,25 @@ def query_elk_uap(index, time_horizon=None): last_end_time[eic] = end_time return result + + +if __name__ == "__main__": + # TESTING + import sys + from emf.common.integrations.opdm import OPDM + logging.basicConfig( + format='%(levelname)-10s %(asctime)s.%(msecs)03d %(name)-30s %(funcName)-35s %(lineno)-5d: %(message)s', + datefmt='%Y-%m-%dT%H:%M:%S', + level=logging.INFO, + handlers=[logging.StreamHandler(sys.stdout)] + ) + #logging.getLogger('powsybl').setLevel(1) + + opdm = OPDM() + # latest_boundary = opdm.get_latest_boundary() + available_models = opdm.get_latest_models_and_download(time_horizon='2D', + scenario_date="2025-06-12T09:30", + tso="AST") + + + opdm_metadata = json.loads(message) diff --git a/emf/model_validator/model_statistics.py b/emf/model_validator/model_statistics.py index 9df04d99..1daa072b 100644 --- a/emf/model_validator/model_statistics.py +++ b/emf/model_validator/model_statistics.py @@ -88,7 +88,7 @@ def parse_type_and_attribute(type_and_attr_string, default_attr="ID"): def get_tieflow_data(data): logger.info("Getting Tieflow Data") tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal->ConnectivityNode") - tieflow_data["BoundaryPoint.isDirectCurrent"] = tieflow_data["IdentifiedObject.description"].str.startswith("HVDC") + tieflow_data["BoundaryPoint.isDirectCurrent"] = tieflow_data["IdentifiedObject.description_ConnectivityNode"].str.startswith("HVDC") # TODO - for CGMES3/CIM17 get also the Boundary objects and use correct field to identify HVDC # Add Injections From b2656a823f486b0cf87e326066b0dc4351253f5a Mon Sep 17 00:00:00 2001 From: Simon <69009411+snec-afk@users.noreply.github.com> Date: Tue, 17 Jun 2025 10:41:30 +0300 Subject: [PATCH 16/67] Update publish-workers.yml --- .github/workflows/publish-workers.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/publish-workers.yml b/.github/workflows/publish-workers.yml index 7abca0fc..fb5649ed 100644 --- a/.github/workflows/publish-workers.yml +++ b/.github/workflows/publish-workers.yml @@ -20,6 +20,7 @@ jobs: - emfos-model-validator - emfos-model-merger - emfos-task-generator + - emfos-model-quality steps: - uses: actions/checkout@v4 From 77b4ed5dbcdd580077e5a4ca016b1bcef7247fac Mon Sep 17 00:00:00 2001 From: Simon <69009411+snec-afk@users.noreply.github.com> Date: Tue, 17 Jun 2025 10:42:59 +0300 Subject: [PATCH 17/67] Update Dockerfile --- docker/Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 233a83c6..a2402610 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -95,3 +95,15 @@ COPY config/ ./config/ ENV PYTHONPATH="${PYTHONPATH}:/src" ENTRYPOINT ["python", "emf/task_generator/worker.py"] #ENTRYPOINT ["/bin/bash"] + +# Model Quality image +FROM balticrcc/emfos-base:latest as emfos-model-quality + +WORKDIR /src + +COPY emf/model_quality/ ./emf/model_quality/ +COPY emf/common/ ./emf/common/ +COPY config/ ./config/ + +ENV PYTHONPATH="${PYTHONPATH}:/src" +ENTRYPOINT ["python", "emf/model_quality/worker.py"] From 04ad706ddeee6f9bc39903da348d763f714c5471 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Tue, 17 Jun 2025 10:44:51 +0300 Subject: [PATCH 18/67] refactor: moving merge pre-process to model validation --- config/cgm_worker/merger.properties | 2 +- .../model_validator.properties | 4 +- emf/common/decorators.py | 3 +- emf/common/integrations/minio_api.py | 24 ++++- .../integrations/object_storage/models.py | 83 +++++++++++++--- emf/common/integrations/rabbit.py | 35 ++++--- emf/common/loadflow_tool/helper.py | 69 +++++++------ emf/model_merger/merge_functions.py | 20 +--- emf/model_merger/model_merger.py | 98 ++++++++----------- emf/model_merger/replacement.py | 12 +-- emf/model_merger/temporary_fixes.py | 66 +------------ emf/model_merger/worker.py | 5 +- emf/model_quality/__init__.py | 0 emf/model_quality/model_quality.py | 5 +- .../model_statistics.py | 0 emf/model_validator/model_validator.py | 75 ++++++++++---- emf/model_validator/validator_functions.py | 33 ++++++- emf/model_validator/worker.py | 1 + 18 files changed, 305 insertions(+), 230 deletions(-) create mode 100644 emf/model_quality/__init__.py rename emf/{model_validator => model_quality}/model_statistics.py (100%) diff --git a/config/cgm_worker/merger.properties b/config/cgm_worker/merger.properties index 5c940c47..270acc26 100644 --- a/config/cgm_worker/merger.properties +++ b/config/cgm_worker/merger.properties @@ -1,12 +1,12 @@ [MAIN] INPUT_RABBIT_QUE = emf-merge.tasks.rmm +OUTPUT_RMQ_EXCHANGE = emf-quality.test INPUT_MINIO_BUCKET = opde-confidential-models INPUT_MINIO_FOLDER = IGM OUTPUT_MINIO_BUCKET = opde-confidential-models OUTPUT_MINIO_FOLDER = EMFOS/RMM MERGE_REPORT_ELK_INDEX = emfos-merge-reports OPDE_MODELS_ELK_INDEX = emfos-opde-models -RABBIT_QUALITY_EXCHANGE = emf-quality.test SMALL_ISLAND_SIZE = 10 MERGE_LOAD_FLOW_SETTINGS = CGM_RELAXED_3_CUSTOM NET_INTERCHANGE_THRESHOLD = 200 diff --git a/config/model_validator/model_validator.properties b/config/model_validator/model_validator.properties index ec606b8e..3ec7c308 100644 --- a/config/model_validator/model_validator.properties +++ b/config/model_validator/model_validator.properties @@ -1,10 +1,10 @@ [MAIN] INPUT_RMQ_QUEUE = object-storage.models.validation +OUTPUT_RMQ_EXCHANGE = emf-quality.test VALIDATION_ELK_INDEX = emfos-igm-validation METADATA_ELK_INDEX = emfos-opde-models ELK_ID_FROM_METADATA_FIELDS = opde:Id,data-source ELK_ID_HASHING = True VALIDATION_LOAD_FLOW_SETTINGS = CGM_RELAXED_2 CHECK_NON_RETAINED_SWITCHES = False -CHECK_KIRCHHOFF_FIRST_LAW = False -RABBIT_QUALITY_EXCHANGE = emf-quality.test \ No newline at end of file +CHECK_KIRCHHOFF_FIRST_LAW = False \ No newline at end of file diff --git a/emf/common/decorators.py b/emf/common/decorators.py index 16c3be60..37eaa4bb 100644 --- a/emf/common/decorators.py +++ b/emf/common/decorators.py @@ -5,6 +5,7 @@ # Start logger logger = logging.getLogger(__name__) + def performance_counter(units='seconds'): """Counts performance of the function""" def decorator_performance_counter(func): @@ -15,7 +16,7 @@ def wrapper_performance_counter(*args, **kwargs): duration = round(time.perf_counter() - start_time, 2) if units == 'minutes': duration = round(duration / 60, 2) # counting by minutes - logger.info(f"Process {func.__name__!r} finished with duration: {duration} {units}", extra={'numeric_value': duration}) + logger.info(f"Process {func.__name__!r} finished with duration: {duration} {units}") return response return wrapper_performance_counter return decorator_performance_counter diff --git a/emf/common/integrations/minio_api.py b/emf/common/integrations/minio_api.py index d48517a3..7fb5ae91 100644 --- a/emf/common/integrations/minio_api.py +++ b/emf/common/integrations/minio_api.py @@ -2,6 +2,7 @@ import requests from lxml import etree import minio +from minio.commonconfig import Tags import urllib3 import sys import mimetypes @@ -90,13 +91,27 @@ def _get_credentials(self, action: str = "AssumeRoleWithLDAPIdentity", version: return credentials + @staticmethod + def dict_to_tags(tags: dict): + converted = Tags.new_object_tags() + for k, v in tags.items(): + converted[k] = v + + return converted + @renew_authentication_token - def upload_object(self, file_path_or_file_object: str | BytesIO, bucket_name: str, metadata: dict | None = None): + def upload_object(self, + file_path_or_file_object: str | BytesIO, + bucket_name: str, + metadata: dict | None = None, + tags: dict | None = None, + ): """ Method to upload file to Minio storage :param file_path_or_file_object: file path or BytesIO object :param bucket_name: bucket name :param metadata: object metadata + :param tags: object tags :return: response from Minio """ file_object = file_path_or_file_object @@ -107,6 +122,10 @@ def upload_object(self, file_path_or_file_object: str | BytesIO, bucket_name: st else: length = file_object.getbuffer().nbytes + # Handle tags if provided + if tags: + tags = self.dict_to_tags(tags) + # Just to be sure that pointer is at the beginning of the content file_object.seek(0) @@ -118,7 +137,8 @@ def upload_object(self, file_path_or_file_object: str | BytesIO, bucket_name: st data=file_object, length=length, content_type=mimetypes.guess_type(file_object.name)[0], - metadata=metadata + metadata=metadata, + tags=tags, ) return response diff --git a/emf/common/integrations/object_storage/models.py b/emf/common/integrations/object_storage/models.py index 657ae99a..77f77e19 100644 --- a/emf/common/integrations/object_storage/models.py +++ b/emf/common/integrations/object_storage/models.py @@ -8,6 +8,74 @@ logger = logging.getLogger(__name__) +def compile_query(metadata: dict, filter: str | None): + + match_and_term_list = [] + for key, value in metadata.items(): + if isinstance(value, list): + match_and_term_list.append({"terms": {key: value}}) + else: + match_and_term_list.append({"match": {key: value}}) + + if filter: + query = {"bool": {"must": match_and_term_list, "filter": {"range": {"pmd:scenarioDate": {"gte": filter}}}}} + else: + query = {"bool": {"must": match_and_term_list}} + + return query + + +def fetch_unique_values(metadata_query: dict, + field: str, + query_filter: str | None = None, + index: str = object_storage.ELASTIC_MODELS_INDEX, + page_size: int = 10000, + ): + + # Validate index definition to be able to search all index by pattern + if "*" not in index: + index = f"{index}*" + + agg_name = "values_page" + unique_vals = [] + after_key = None + + # Compile query + query = compile_query(metadata=metadata_query, filter=query_filter) + + while True: + comp = { + "size": page_size, + "sources": [ + {"val": {"terms": {"field": field}}} + ] + } + if after_key: + comp["after"] = after_key + + body = { + "size": 0, + "query": query, + "aggs": { + agg_name: { + "composite": comp + } + } + } + + response = object_storage.elastic_service.client.search(index=index, body=body) + buckets = response["aggregations"][agg_name]["buckets"] + unique_vals.extend(b["key"]["val"] for b in buckets) + + # Page on if there's more + if "after_key" in response["aggregations"][agg_name]: + after_key = response["aggregations"][agg_name]["after_key"] + else: + break + + return unique_vals + + def query_data(metadata_query: dict, query_filter: str | None = None, index: str = object_storage.ELASTIC_MODELS_INDEX, @@ -53,23 +121,14 @@ def query_data(metadata_query: dict, if "*" not in index: index = f"{index}*" - match_and_term_list = [] - for key, value in metadata_query.items(): - if isinstance(value, list): - match_and_term_list.append({"terms": {key: value}}) - else: - match_and_term_list.append({"match": {key: value}}) - - if query_filter: - query = {"bool": {"must": match_and_term_list, "filter": {"range": {"pmd:scenarioDate": {"gte": query_filter}}}}} - else: - query = {"bool": {"must": match_and_term_list}} + # Compile query + query = compile_query(metadata=metadata_query, filter=query_filter) # Return query results response = object_storage.elastic_service.client.search(index=index, query=query, size=size, sort=sort, scroll=scroll) scroll_id = response['_scroll_id'] hits = response["hits"]["hits"] - content_list = [content["_source"] for content in hits] + content_list = [content["_source"] for content in hits] # TODO - consider optimize without this loop while len(hits) > 0: response = object_storage.elastic_service.client.scroll(scroll_id=scroll_id, scroll=scroll) hits = response["hits"]["hits"] diff --git a/emf/common/integrations/rabbit.py b/emf/common/integrations/rabbit.py index 557f420c..495da745 100644 --- a/emf/common/integrations/rabbit.py +++ b/emf/common/integrations/rabbit.py @@ -6,6 +6,7 @@ from typing import List from emf.common.config_parser import parse_app_properties from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor + # from pika.adapters.asyncio_connection import AsyncioConnection # import asyncio @@ -93,7 +94,8 @@ def consume_start(self, queue: str, callback: object | None = None, auto_ack: bo # Set up consumer if not callback: - callback = lambda ch, method, properties, body: logger.info(f"Received message: {properties} (No callback processing)") + callback = lambda ch, method, properties, body: logger.info( + f"Received message: {properties} (No callback processing)") self.consume_channel.basic_consume( queue=queue, @@ -174,7 +176,7 @@ def __init__(self, port: int = int(RMQ_PORT), vhost: str = RMQ_VHOST, queue: str | None = None, - reply_to: str | None = None, + forward: str | None = None, username: str = RMQ_USERNAME, password: str = RMQ_PASSWORD, heartbeat: str | int = RMQ_HEARTBEAT_IN_SEC, @@ -183,14 +185,14 @@ def __init__(self, """Create a new instance of the consumer class, passing in the AMQP URL used to connect to RabbitMQ. - If 'reply_to' is provided messaged will be published to given queue name + If 'forward' is provided messaged will be published to given queue/exchange name """ self.message_handlers = message_handlers self.message_converter = message_converter self.should_reconnect = False self.was_consuming = False - self.reply_to = reply_to + self.forward = forward self._connection = None self._channel = None @@ -212,7 +214,8 @@ def __init__(self, port=self._port, virtual_host=self._vhost, credentials=pika.PlainCredentials(username, password), - heartbeat= None if RMQ_HEARTBEAT_IN_SEC == "0" else int(RMQ_HEARTBEAT_IN_SEC)) + heartbeat=None if RMQ_HEARTBEAT_IN_SEC == "0" else int( + RMQ_HEARTBEAT_IN_SEC)) def connect(self): """This method connects to RabbitMQ, returning the connection handle. @@ -223,7 +226,7 @@ def connect(self): """ logger.info(f"Connecting to {self._host}:{self._port} @ {self._vhost} as {self._username}") - + return pika.SelectConnection( parameters=self._connection_parameters, on_open_callback=self.on_connection_open, @@ -397,12 +400,11 @@ def _process_messages(self, basic_deliver, properties, body): # self.connection.close() # self.stop() - # Publish message to next exchange/queue if provided - if self.reply_to: - logger.info(f"Publishing message to exchange/queue: {self.reply_to}") - self._channel.basic_publish(exchange='', routing_key=self.reply_to, body=body, properties=properties) - if ack: + # Publish message to next exchange/queue if provided and message successfully handled + if self.forward: + logger.info(f"Publishing message to exchange: {self.forward}") + self._channel.basic_publish(exchange=self.forward, routing_key="", body=body, properties=properties) self.acknowledge_message(basic_deliver.delivery_tag) def on_message(self, _unused_channel, basic_deliver, properties, body): @@ -417,7 +419,8 @@ def on_message(self, _unused_channel, basic_deliver, properties, body): :param pika.Spec.BasicProperties: properties :param bytes body: The message body """ - logger.info(f"Received message # {basic_deliver.delivery_tag} from {properties.app_id} meta: {properties.headers}") + logger.info( + f"Received message # {basic_deliver.delivery_tag} from {properties.app_id} meta: {properties.headers}") logger.debug(f"Message body: {body}") self._executor.submit(self._process_messages, basic_deliver, properties, body) @@ -494,12 +497,13 @@ class ReconnectingConsumer: """This is an example consumer that will reconnect if the nested RMQConsumer indicates that a reconnect is necessary. """ + def __init__(self, host: str = RMQ_SERVER, port: int = int(RMQ_PORT), vhost: str = RMQ_VHOST, queue: str | None = None, - reply_to: str | None = None, + forward: str | None = None, username: str = RMQ_USERNAME, password: str = RMQ_PASSWORD, message_handler: object | None = None, @@ -509,7 +513,7 @@ def __init__(self, self._port = port self._vhost = vhost self._queue = queue - self._reply_to = reply_to + self._forward = forward self._username = username self.__password = password self.message_handler = message_handler @@ -518,7 +522,7 @@ def __init__(self, port=self._port, vhost=self._vhost, queue=self._queue, - reply_to=self._reply_to, + forward=self._forward, username=self._username, password=self.__password, message_handlers=self.message_handler, @@ -557,6 +561,7 @@ def stop(self): if __name__ == '__main__': # Testing RMQ API import sys + logging.basicConfig(stream=sys.stdout, format="%(levelname) -10s %(asctime) -10s %(name) -35s %(funcName) -30s %(lineno) -5d: %(message)s", level=logging.INFO) diff --git a/emf/common/loadflow_tool/helper.py b/emf/common/loadflow_tool/helper.py index a94cf037..c297a124 100644 --- a/emf/common/loadflow_tool/helper.py +++ b/emf/common/loadflow_tool/helper.py @@ -15,6 +15,7 @@ import uuid from aniso8601 import parse_datetime import re +import config logger = logging.getLogger(__name__) @@ -99,11 +100,11 @@ def create_opdm_objects(models: list, metadata=None, key_profile="SV") -> list: opdm_profile = opdm_metadata_from_filename(profile_instance.name) opdm_profile.update(opdm_metadata_from_rdfxml(get_xml_from_zip(profile_instance))) opdm_profile['pmd:fileName'] = profile_instance.name - opdm_profile["pmd:content-reference"] = generate_OPDM_ContentReference_from_filename(profile_instance.name) + opdm_profile["pmd:content-reference"] = generate_opdm_content_reference_from_filename(profile_instance.name) # Check if key profile and add to main object metadata if opdm_profile.get('pmd:cgmesProfile') == key_profile: - opdm_object.update(opdm_profile) + opdm_object.update(opdm_profile) # Add model type - IGM/CGM # TODO develop logic to also define as CGM metadata object @@ -240,7 +241,6 @@ def get_network_elements(network: pypowsybl.network, def get_slack_generators(network: pypowsybl.network): - slack_terminals = network.get_extension('slackTerminal') slack_generators = get_network_elements(network=network, element_type=pypowsybl.network.ElementType.GENERATOR, @@ -320,7 +320,6 @@ def get_opdm_data_from_models(model_data: list | pandas.DataFrame): def filename_from_opdm_metadata(metadata): - model_part = metadata.get('pmd:modelPartReference', None) if model_part: @@ -381,7 +380,7 @@ def opdm_metadata_from_filename(file_name: str, meta_separator: str = "_"): elif len(meta_list) == 5: metadata[VALID_FROM], metadata[TIME_HORIZON], model_authority, metadata[CGMES_PROFILE], metadata[VERSION_NUMBER] = meta_list else: - logger.warning(f"Parsing error, number of allowed meta in filename is 4 or 5 separated by {meta_separator} -> {file_name}") + logger.warning(f"Parsing error, number of allowed meta in filename is 4 or 5 separated by {meta_separator}: {file_name}") return metadata # Parse model authority components @@ -400,7 +399,7 @@ def opdm_metadata_from_filename(file_name: str, meta_separator: str = "_"): if metadata.get(MODEL_PART): metadata[TSO] = metadata[MODEL_PART] metadata[SOURCING_ACTOR] = metadata[MODEL_PART] - + return metadata @@ -411,7 +410,7 @@ def metadata_from_rdfxml(parsed_xml: etree._ElementTree): rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" - header = parsed_xml.find("{*}FullModel") # Verion update proof, as long as element name remains the same + header = parsed_xml.find("{*}FullModel") # Version update proof, as long as element name remains the same meta_elements = header.getchildren() # Add model ID from FullModel@about @@ -433,7 +432,6 @@ def metadata_from_rdfxml(parsed_xml: etree._ElementTree): def opdm_metadata_from_rdfxml(parsed_xml: etree._ElementTree): - opdm_metadata_map = { "Model.mRID": ["opde:Id", "pmd:modelid", "pmd:fullModel_ID"], @@ -464,7 +462,6 @@ def opdm_metadata_from_rdfxml(parsed_xml: etree._ElementTree): def get_xml_from_zip(zip_file_path): - with ZipFile(zip_file_path, 'r') as zipfile_object: xml_file_name = zipfile_object.namelist()[0] file_bytes = zipfile_object.read(xml_file_name) @@ -475,7 +472,6 @@ def get_xml_from_zip(zip_file_path): def zip_xml(xml_file_object): - xml_file_name = xml_file_object.name xml_file_extension = xml_file_name.split(".")[-1] zip_file_name = xml_file_name.replace(f".{xml_file_extension}", ".zip") @@ -491,7 +487,6 @@ def zip_xml(xml_file_object): def get_metadata_from_filename(file_name): - # Separators file_type_separator = "." meta_separator = "_" @@ -507,9 +502,9 @@ def get_metadata_from_filename(file_name): # Naming before QoDC 2.1, where EQ might not have processType if len(file_meta_list) == 4: - file_metadata["Model.scenarioTime"],\ - file_metadata["Model.modelingEntity"],\ - file_metadata["Model.messageType"],\ + file_metadata["Model.scenarioTime"], \ + file_metadata["Model.modelingEntity"], \ + file_metadata["Model.messageType"], \ file_metadata["Model.version"] = file_meta_list file_metadata["Model.processType"] = "" @@ -518,11 +513,11 @@ def get_metadata_from_filename(file_name): # Naming after QoDC 2.1, always 5 positions elif len(file_meta_list) == 5: - file_metadata["Model.scenarioTime"],\ - file_metadata["Model.processType"],\ - file_metadata["Model.modelingEntity"],\ - file_metadata["Model.messageType"],\ - file_metadata["Model.version"] = file_meta_list + file_metadata["Model.scenarioTime"], \ + file_metadata["Model.processType"], \ + file_metadata["Model.modelingEntity"], \ + file_metadata["Model.messageType"], \ + file_metadata["Model.version"] = file_meta_list else: logger.error("Non CGMES file {}".format(file_name)) @@ -532,24 +527,24 @@ def get_metadata_from_filename(file_name): entity_and_area_list = file_metadata["Model.modelingEntity"].split(entity_and_domain_separator) if len(entity_and_area_list) == 1: - file_metadata["Model.mergingEntity"],\ - file_metadata["Model.domain"] = "", "" # Set empty string for both + file_metadata["Model.mergingEntity"], \ + file_metadata["Model.domain"] = "", "" # Set empty string for both file_metadata["Model.forEntity"] = entity_and_area_list[0] if len(entity_and_area_list) == 2: - file_metadata["Model.mergingEntity"],\ + file_metadata["Model.mergingEntity"], \ file_metadata["Model.domain"] = entity_and_area_list file_metadata["Model.forEntity"] = "" if len(entity_and_area_list) == 3: - file_metadata["Model.mergingEntity"],\ - file_metadata["Model.domain"],\ + file_metadata["Model.mergingEntity"], \ + file_metadata["Model.domain"], \ file_metadata["Model.forEntity"] = entity_and_area_list return file_metadata -def generate_OPDM_ContentReference_from_filename(file_name, opdm_object_type="CGMES"): +def generate_opdm_content_reference_from_filename(file_name: str, opdm_object_type="CGMES"): """ Generates the file path based on the given parameters and metadata extracted from the filename. @@ -560,11 +555,6 @@ def generate_OPDM_ContentReference_from_filename(file_name, opdm_object_type="CG Returns: str: Generated file path using the provided template and extracted metadata. - Example: - >>> file_name = "example_filename" - >>> opdm_object_type = "CGMES" - >>> generate_OPDM_ContentReference_from_filename(file_name, opdm_object_type) - 'CGMES/processType/modelingEntity/20240529/123000/messageType/example_filename' """ template = "{opdm_object_type}/{processType}/{modelingEntity}/{scenarioTime:%Y%m%d}/{scenarioTime:%H%M00}/{messageType}/{file_name}" @@ -600,6 +590,25 @@ def export_model(network: pypowsybl.network, opdm_object_meta, profiles=None): return bytes_object +def export_to_cgmes_zip(triplets: list): + namespace_map = { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "cim": "http://iec.ch/TC57/2013/CIM-schema-cim16#", + "md": "http://iec.ch/TC57/61970-552/ModelDescription/1#", + "entsoe": "http://entsoe.eu/CIM/SchemaExtension/3/1#", + } + + # with open('../../config/cgm_worker/CGMES_v2_4_15_2014_08_07.json', 'r') as file_object: + rdf_map = json.load(config.paths.cgm_worker.CGMES_v2_4_15_2014_08_07) + + return pandas.concat(triplets, ignore_index=True).export_to_cimxml(rdf_map=rdf_map, + namespace_map=namespace_map, + export_undefined=False, + export_type="xml_per_instance_zip_per_xml", + debug=False, + export_to_memory=True) + + def get_model_outages(network: pypowsybl.network): # Get network elements diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index 3fa69e61..cdb68042 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) SV_INJECTION_LIMIT = 0.1 + def is_valid_uuid(uuid_value): """ Checks if input is uuid value @@ -464,25 +465,6 @@ def configure_paired_boundarypoint_injections_by_nodes(data): return data.update_triplet_from_triplet(pandas.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) -def export_to_cgmes_zip(triplets: list): - namespace_map = { - "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", - "cim": "http://iec.ch/TC57/2013/CIM-schema-cim16#", - "md": "http://iec.ch/TC57/61970-552/ModelDescription/1#", - "entsoe": "http://entsoe.eu/CIM/SchemaExtension/3/1#", - } - - # with open('../../config/cgm_worker/CGMES_v2_4_15_2014_08_07.json', 'r') as file_object: - rdf_map = json.load(config.paths.cgm_worker.CGMES_v2_4_15_2014_08_07) - - return pandas.concat(triplets, ignore_index=True).export_to_cimxml(rdf_map=rdf_map, - namespace_map=namespace_map, - export_undefined=False, - export_type="xml_per_instance_zip_per_xml", - debug=False, - export_to_memory=True) - - def generate_merge_report(merged_model: object, task: dict): """ Creates JSON type report of pypowsybl loadflow results diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index c74b4d2b..f4d46943 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -17,17 +17,16 @@ from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download from emf.common.integrations.object_storage.schedules import query_acnp_schedules, query_hvdc_schedules from emf.common.loadflow_tool import loadflow_settings -from emf.common.loadflow_tool.helper import opdmprofile_to_bytes, attr_to_dict +from emf.common.loadflow_tool.helper import opdmprofile_to_bytes, attr_to_dict, export_to_cgmes_zip from emf.model_merger import merge_functions from emf.model_merger import scaler -from emf.model_merger.replacement import run_replacement, get_available_tsos +from emf.model_merger.replacement import run_replacement, get_tsos_available_in_storage from emf.task_generator.task_generator import update_task_status from emf.common.logging.custom_logger import get_elk_logging_handler # TODO - move this async solution to some common module from concurrent.futures import ThreadPoolExecutor from lxml import etree -from emf.model_merger.temporary_fixes import run_post_merge_processing, run_pre_merge_processing, \ - fix_model_outages, open_switches_in_network, fix_igm_ssh_vs_cgm_ssh_error +from emf.model_merger.temporary_fixes import run_post_merge_processing, fix_model_outages, fix_igm_ssh_vs_cgm_ssh_error logger = logging.getLogger(__name__) parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.merger) @@ -139,6 +138,8 @@ def handle(self, task_object: dict, properties: dict, **kwargs): post_temp_fixes = task_properties['post_temp_fixes'] force_outage_fix = task_properties['force_outage_fix'] open_non_retained_switches_between_tn = json.loads(OPEN_NON_RETAINED_SWITCHES_BETWEEN_TN.lower()) + + # TODO used for temp fixes - consider move it out somewhere try: net_interchange_threshold = int(NET_INTERCHANGE_THRESHOLD) except Exception: @@ -146,7 +147,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): task_properties['net_interchange2_threshold'] = net_interchange_threshold - remove_non_generators_from_slack_participation = True + remove_non_generators_from_slack_participation = True # TODO move to conf # Collect valid models from ObjectStorage downloaded_models = get_latest_models_and_download(time_horizon, scenario_datetime, valid=False, data_source='OPDM') @@ -193,17 +194,20 @@ def handle(self, task_object: dict, properties: dict, **kwargs): valid_models = [model for model in filtered_models if model['valid']] invalid_models = [model['pmd:TSO'] for model in filtered_models if model not in valid_models] if invalid_models: - merged_model.excluded.extend([{'tso': tso, 'reason': 'Invalid'} for tso in invalid_models]) + merged_model.excluded.extend([{'tso': tso, 'reason': 'invalid'} for tso in invalid_models]) # Check missing models for replacement if included_models: missing_models = [model for model in included_models if model not in [model['pmd:TSO'] for model in valid_models]] if missing_models: - merged_model.excluded.extend([{'tso': tso, 'reason': 'Missing in OPDM'} for tso in missing_models]) + merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-opdm'} for tso in missing_models]) else: if model_replacement: - available_tsos = get_available_tsos() - missing_models = [model for model in available_tsos if model not in [model['pmd:TSO'] for model in valid_models] + excluded_models] + # Get TSOs who models are available in storage for replacement period + available_tsos = get_tsos_available_in_storage() + valid_model_tsos = [model['pmd:TSO'] for model in valid_models] + # Need to ensure that excluded models by task configuration would not be taken in replacement context + missing_models = [tso for tso in available_tsos if tso not in valid_model_tsos + excluded_models] else: missing_models = [] @@ -241,14 +245,6 @@ def handle(self, task_object: dict, properties: dict, **kwargs): logger.warning("Found no models to merge, returning None") return None - # Run pre-processing - pre_p_start = datetime.datetime.now(datetime.UTC) - between_tn = pandas.DataFrame() - if pre_temp_fixes: - input_models, between_tn = run_pre_merge_processing(input_models, merging_area) - pre_p_end = datetime.datetime.now(datetime.UTC) - logger.debug(f"Pre-processing took: {(pre_p_end - pre_p_start).total_seconds()} seconds") - # Load network model and merge merge_start = datetime.datetime.now(datetime.UTC) merged_model.network = merge_functions.load_model(opdm_objects=input_models) @@ -261,11 +257,11 @@ def handle(self, task_object: dict, properties: dict, **kwargs): # Update model outages tso_list = [] - if force_outage_fix: #force outage fix on all models if set + if force_outage_fix: # force outage fix on all models if set tso_list = valid_tso_list - elif merging_area == 'BA' and any(tso in ['LITGRID', 'AST', 'ELERING'] for tso in replaced_tso_list): #by default do it on Baltic merge replaced models + elif merging_area == 'BA' and any(tso in ['LITGRID', 'AST', 'ELERING'] for tso in replaced_tso_list): # by default do it on Baltic merge replaced models tso_list = replaced_tso_list - if tso_list: #if not set force and not replaced BA then nothing to fix + if tso_list: # if not set force and not replaced BA then nothing to fix merged_model = fix_model_outages(merged_model=merged_model, tso_list=tso_list, scenario_datetime=scenario_datetime, @@ -275,10 +271,9 @@ def handle(self, task_object: dict, properties: dict, **kwargs): if remove_non_generators_from_slack_participation: merged_model.network = fix_igm_ssh_vs_cgm_ssh_error(merged_model.network) - if open_non_retained_switches_between_tn and isinstance(between_tn, pandas.DataFrame): - if not between_tn.empty: - merged_model.network = open_switches_in_network(network_pre_instance=merged_model.network, - switches_dataframe=between_tn) + # TODO - placeholder for open TN switcher + + # TODO placeholder for configure paired tie lines # TODO - run other LF if default fails # Run loadflow on merged model @@ -296,9 +291,6 @@ def handle(self, task_object: dict, properties: dict, **kwargs): if not schedule_start: schedule_start = scenario_datetime - # Print out the schedule parameters for scaling - logger.info(f"Querying schedules for: {schedule_time_horizon} {schedule_start}") - # Get aligned schedules ac_schedules = query_acnp_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) dc_schedules = query_hvdc_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) @@ -329,15 +321,17 @@ def handle(self, task_object: dict, properties: dict, **kwargs): # Run post-processing post_p_start = datetime.datetime.now(datetime.UTC) + logger.info(f"Starting merged model post-processing") + # TODO here should be one existing network structure. IIDM model can be exported and removed to release memory sv_data, ssh_data, opdm_object_meta = run_post_merge_processing(input_models=input_models, - merged_model=merged_model, - task_properties=task_properties, - small_island_size=SMALL_ISLAND_SIZE, - enable_temp_fixes=post_temp_fixes, - time_horizon=new_time_horizon) + merged_model=merged_model, + task_properties=task_properties, + small_island_size=SMALL_ISLAND_SIZE, + enable_temp_fixes=post_temp_fixes, + time_horizon=new_time_horizon) # Package both input models and exported CGM profiles to in memory zip files - serialized_data = merge_functions.export_to_cgmes_zip([ssh_data, sv_data]) + serialized_data = export_to_cgmes_zip([ssh_data, sv_data]) post_p_end = datetime.datetime.now(datetime.UTC) logger.debug(f"Post processing took: {(post_p_end - post_p_start).total_seconds()} seconds") logger.debug(f"Merging took: {(merge_end - merge_start).total_seconds()} seconds") @@ -352,8 +346,8 @@ def handle(self, task_object: dict, properties: dict, **kwargs): callback=log_opdm_response, file_path_or_file_object=item) merged_model.uploaded_to_opde = True - except: - logging.error(f"""Unexpected error on uploading to OPDM:""", exc_info=True) + except Exception as error: + logging.error(f"Unexpected error on uploading to OPDM: {error}", exc_info=True) # Create zipped model data if merging_area == 'BA': @@ -382,11 +376,14 @@ def handle(self, task_object: dict, properties: dict, **kwargs): logger.info(f"Uploading merged model to MINIO: {merged_model_object.name}") minio_metadata = merge_functions.evaluate_trustability(merged_model.__dict__, task['task_properties']) try: - response = self.minio_service.upload_object(merged_model_object, bucket_name=OUTPUT_MINIO_BUCKET, metadata=minio_metadata) + response = self.minio_service.upload_object(file_path_or_file_object=merged_model_object, + bucket_name=OUTPUT_MINIO_BUCKET, + metadata=minio_metadata, + ) if response: merged_model.uploaded_to_minio = True - except: - logging.error(f"""Unexpected error on uploading to Object Storage:""", exc_info=True) + except Exception as error: + logging.error(f"Unexpected error on uploading to Object Storage: {error}", exc_info=True) logger.info(f"Merged model creation done for: {merged_model.name}") @@ -406,8 +403,9 @@ def handle(self, task_object: dict, properties: dict, **kwargs): merged_model.duration_s = (end_time - merge_start).total_seconds() merged_model.content_reference = merged_model_object.name - # Update OPDM object data with CGM relevant data + # Update OPDM object data with CGM relevant data and send to Elastic opdm_object_meta = merge_functions.update_cgm_opdm_object_meta(opdm_object_meta, merged_model) + response = elastic.Elastic.send_to_elastic(index=OPDE_MODELS_ELK_INDEX, json_message=opdm_object_meta) # Send merge report and opdm object metadata to Elastic if model_merge_report_send_to_elk: @@ -416,27 +414,17 @@ def handle(self, task_object: dict, properties: dict, **kwargs): merge_report = merge_functions.generate_merge_report(merged_model=merged_model, task=task) try: response = elastic.Elastic.send_to_elastic(index=MERGE_REPORT_ELK_INDEX, json_message=merge_report) - response2 = elastic.Elastic.send_to_elastic(index=OPDE_MODELS_ELK_INDEX, json_message=opdm_object_meta) except Exception as error: logger.error(f"Merge report sending to Elastic failed: {error}") except Exception as error: logger.error(f"Failed to create merge report: {error}") - # Send model metadata to quality assurance module - if model_upload_to_minio and model_merge_report_send_to_elk: - try: - merge_report['minio-bucket'] = OUTPUT_MINIO_BUCKET - rabbit.BlockingClient().publish(payload=json.dumps(merge_report), exchange_name=RABBIT_QUALITY_EXCHANGE, - headers={"opde:Object-Type": "CGM"}) - except: - logger.error(f"Sending metadata to RabbitMQ failed: {error}") - # Stop Trace self.elk_logging_handler.stop_trace() logger.info(f"Merge task finished for model: {merged_model.name}") - return task, properties + return opdm_object_meta, properties if __name__ == "__main__": @@ -469,14 +457,14 @@ def handle(self, task_object: dict, properties: dict, **kwargs): "job_period_start": "2024-05-24T22:00:00+00:00", "job_period_end": "2024-05-25T06:00:00+00:00", "task_properties": { - "timestamp_utc": "2025-06-01T12:30:00+00:00", - "merge_type": "BA", + "timestamp_utc": "2025-06-12T12:30:00+00:00", + "merge_type": "EU", "merging_entity": "BALTICRCC", - "included": ["LITGRID"], + "included": [], "excluded": [], "local_import": [], - "time_horizon": "2D", - "version": "00", + "time_horizon": "1D", + "version": "001", "mas": "http://www.baltic-rsc.eu/OperationalPlanning", "pre_temp_fixes": "True", "post_temp_fixes": "True", diff --git a/emf/model_merger/replacement.py b/emf/model_merger/replacement.py index e14000d5..1f46cb57 100644 --- a/emf/model_merger/replacement.py +++ b/emf/model_merger/replacement.py @@ -6,7 +6,7 @@ import json from dateutil import parser from pathlib import Path -from emf.common.integrations.object_storage.models import query_data, get_content +from emf.common.integrations.object_storage.models import query_data, get_content, fetch_unique_values from emf.common.integrations.minio_api import * from emf.common.config_parser import parse_app_properties @@ -196,11 +196,11 @@ def create_replacement_table(target_timestamp, target_timehorizon, valid_models_ return valid_models_df -def get_available_tsos(): - query = {"opde:Object-Type": "IGM", "valid": True} - body = query_data(query, QUERY_FILTER) - key = 'pmd:TSO' - return list({item[key] for item in body if key in item}) +def get_tsos_available_in_storage(): + metadata = {"opde:Object-Type": "IGM", "valid": True} + unique_tsos = fetch_unique_values(metadata_query=metadata, field="pmd:TSO.keyword", query_filter=QUERY_FILTER) + + return unique_tsos def get_first_monday_of_last_month(timestamp): diff --git a/emf/model_merger/temporary_fixes.py b/emf/model_merger/temporary_fixes.py index a74e9d76..897faeda 100644 --- a/emf/model_merger/temporary_fixes.py +++ b/emf/model_merger/temporary_fixes.py @@ -4,11 +4,10 @@ import logging from emf.common.integrations import elastic -from emf.common.loadflow_tool.helper import create_opdm_objects, get_model_outages, get_network_elements, get_opdm_data_from_models -from emf.model_merger.merge_functions import (load_opdm_data, create_sv_and_updated_ssh, fix_sv_shunts, +from emf.common.loadflow_tool.helper import get_model_outages, get_network_elements, get_opdm_data_from_models, load_opdm_data +from emf.model_merger.merge_functions import (create_sv_and_updated_ssh, fix_sv_shunts, fix_sv_tapsteps, remove_duplicate_sv_voltages, remove_small_islands, check_and_fix_dependencies, - export_to_cgmes_zip, configure_paired_boundarypoint_injections_by_nodes, ) @@ -16,47 +15,6 @@ logger = logging.getLogger(__name__) -def check_switch_terminals(input_data: pd.DataFrame, column_name: str): - """ - Checks if column of a dataframe contains only one value - :param input_data: input data frame - :param column_name: name of the column to check - return True if different values are in column, false otherwise - """ - data_slice = (input_data.reset_index())[column_name] - return not pd.Series(data_slice[0] == data_slice).all() - - -def get_not_retained_switches_between_nodes(original_data): - """ - For the loadflow open all the non-retained switches that connect different topological nodes - Currently it is seen to help around 9 to 10 Kirchhoff 1st law errors from 2 TSOs - :param original_data: original models in triplets format - :return: updated original data - """ - updated_switches = False - original_models = get_opdm_data_from_models(original_data) - not_retained_switches = original_models[(original_models['KEY'] == 'Switch.retained') - & (original_models['VALUE'] == "false")][['ID']] - closed_switches = original_models[(original_models['KEY'] == 'Switch.open') - & (original_models['VALUE'] == 'false')] - not_retained_closed = not_retained_switches.merge(closed_switches[['ID']], on='ID') - terminals = original_models.type_tableview('Terminal').rename_axis('Terminal').reset_index() - terminals = terminals[['Terminal', - # 'ACDCTerminal.connected', - 'Terminal.ConductingEquipment', - 'Terminal.TopologicalNode']] - not_retained_terminals = (terminals.rename(columns={'Terminal.ConductingEquipment': 'ID'}) - .merge(not_retained_closed, on='ID')) - if not_retained_terminals.empty: - return original_data, updated_switches - between_tn = ((not_retained_terminals.groupby('ID')[['Terminal.TopologicalNode']] - .apply(lambda x: check_switch_terminals(x, 'Terminal.TopologicalNode'))) - .reset_index(name='same_TN')) - between_tn = between_tn[between_tn['same_TN']] - return between_tn - - def open_switches_in_network(network_pre_instance: pypowsybl.network.Network, switches_dataframe: pd.DataFrame): """ Opens switches in loaded network given bu dataframe (uses ID for merging) @@ -73,23 +31,6 @@ def open_switches_in_network(network_pre_instance: pypowsybl.network.Network, sw return network_pre_instance -def run_pre_merge_processing(input_models, merging_area): - - # TODO warning logs for temp fix functions - assembled_data = load_opdm_data(input_models) - assembled_data = triplets.cgmes_tools.update_FullModel_from_filename(assembled_data) - assembled_data = configure_paired_boundarypoint_injections_by_nodes(assembled_data) - escape_upper_xml = assembled_data[assembled_data['VALUE'].astype(str).str.contains('.XML')] - between_tn = get_not_retained_switches_between_nodes(assembled_data) - if not escape_upper_xml.empty: - escape_upper_xml['VALUE'] = escape_upper_xml['VALUE'].str.replace('.XML', '.xml') - assembled_data = triplets.rdf_parser.update_triplet_from_triplet(assembled_data, escape_upper_xml, update=True, add=False) - - input_models = create_opdm_objects([export_to_cgmes_zip([assembled_data])]) - - return input_models, between_tn - - def check_net_interchanges(cgm_sv_data, cgm_ssh_data, original_models, fix_errors: bool = False, threshold: float = None): """ @@ -263,7 +204,6 @@ def check_all_kind_of_injections(cgm_sv_data, filtered = pd.concat(filtered_list).drop_duplicates().reset_index(drop=True) if not filtered.empty: logger.warning(f"Found {len(filtered.index)} mismatches between {injection_name} and flow values on terminals") - print(filtered.to_string()) if fix_errors: logger.info(f"Updating {injection_name} values from terminal flow values") injections_update = injections.merge(filtered[[*fixed_fields, *fields_to_check.keys()]]) @@ -305,7 +245,7 @@ def run_post_merge_processing(input_models: list, sv_data = remove_small_islands(sv_data, int(small_island_size)) sv_data = remove_duplicate_sv_voltages(cgm_sv_data=sv_data, original_data=models_as_triplets) sv_data = check_and_fix_dependencies(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=models_as_triplets) - #sv_data, ssh_data = disconnect_equipment_if_flow_sum_not_zero(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=models_as_triplets) fix implemented in pypowsybl 1.8.1 + #sv_data, ssh_data = disconnect_equipment_if_flow_sum_not_zero(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=models_as_triplets) fix implemented in pypowsybl 1.8.1 ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, cgm_sv_data=sv_data, original_models=models_as_triplets, diff --git a/emf/model_merger/worker.py b/emf/model_merger/worker.py index f052cae0..83d8b787 100644 --- a/emf/model_merger/worker.py +++ b/emf/model_merger/worker.py @@ -17,7 +17,10 @@ parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.merger) # RabbitMQ consumer implementation -consumer = rabbit.RMQConsumer(queue=INPUT_RABBIT_QUE, message_handlers=[HandlerMergeModels()]) +consumer = rabbit.RMQConsumer(queue=INPUT_RABBIT_QUE, + message_handlers=[HandlerMergeModels()], + forward=OUTPUT_RMQ_EXCHANGE, + ) try: consumer.run() diff --git a/emf/model_quality/__init__.py b/emf/model_quality/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/emf/model_quality/model_quality.py b/emf/model_quality/model_quality.py index efa23ff0..0e809db6 100644 --- a/emf/model_quality/model_quality.py +++ b/emf/model_quality/model_quality.py @@ -4,13 +4,12 @@ import json import pypowsybl as pp from emf.common.config_parser import parse_app_properties -from emf.common.integrations import elastic, minio_api, rabbit +from emf.common.integrations import elastic, minio_api from emf.common.integrations.object_storage import models from io import BytesIO from zipfile import ZipFile, ZIP_DEFLATED -from emf.common.loadflow_tool.helper import get_model_outages from triplets.rdf_parser import load_all_to_dataframe -from emf.model_validator.model_statistics import get_system_metrics +from emf.model_quality.model_statistics import get_system_metrics logger = logging.getLogger(__name__) diff --git a/emf/model_validator/model_statistics.py b/emf/model_quality/model_statistics.py similarity index 100% rename from emf/model_validator/model_statistics.py rename to emf/model_quality/model_statistics.py diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index f12c7a3f..e83d4eff 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -6,13 +6,15 @@ import math import pypowsybl as pp import uuid +import triplets from emf.common.loadflow_tool.helper import attr_to_dict, load_model from emf.common.config_parser import parse_app_properties from emf.common.integrations import elastic, minio_api, rabbit from emf.common.integrations.object_storage import models -from emf.common.loadflow_tool.helper import load_opdm_data, clean_data_from_opdm_objects +from emf.common.loadflow_tool.helper import load_opdm_data, clean_data_from_opdm_objects, export_to_cgmes_zip from emf.common.loadflow_tool import loadflow_settings from emf.model_validator import validator_functions +from emf.common.decorators import performance_counter logger = logging.getLogger(__name__) @@ -124,6 +126,39 @@ def run_validation(self): self.validate_kirchhoff_first_law() +class TemporaryPreMergeModifications: + """Run pre-processing modification important for successful merge process""" + # TODO needs to keep in mind that this is meant as temporary fixes and needs to be revised often + def __init__(self, network: pd.DataFrame): + self.network = network + self.report = {'modification': {}} + + def update_header_from_file_name(self): + self.report['modification']['header_from_file_name'] = False + modified_data = triplets.cgmes_tools.update_FullModel_from_filename(self.network) + if not pd.concat([self.network, modified_data]).drop_duplicates(keep=False).empty: + self.network = modified_data + self.report['modification']['header_from_file_name'] = True + + def sanitize_file_name(self): + self.report['modification']['sanitize_file_name'] = False + xml_extension_typo = self.network[self.network['VALUE'].str.contains('.XML', regex=False, na=False)] + if not xml_extension_typo.empty: + xml_extension_typo['VALUE'] = xml_extension_typo['VALUE'].str.replace('.XML', '.xml') + self.network = triplets.rdf_parser.update_triplet_from_triplet(data=self.network, + update_data=xml_extension_typo, + update=True, + add=False) + self.report['modification']['sanitize_file_name'] = True + + @performance_counter(units='seconds') + def run_pre_process_modifications(self): + self.update_header_from_file_name() + self.sanitize_file_name() + + return self.network + + class HandlerModelsValidator: def __init__(self): @@ -165,9 +200,26 @@ def handle(self, message: bytes, properties: dict, **kwargs): post_lf_validation = PostLFValidator(network=network, network_triplets=network_triplets) post_lf_validation.run_validation() + # Clean opdm object from DATA as this is already converted to other formats + opdm_object = clean_data_from_opdm_objects(opdm_objects=[opdm_object])[0] + + # Apply pre-processing modification to models and store in Minio + pre_merge_modification = TemporaryPreMergeModifications(network=network_triplets) + network_triplets = pre_merge_modification.run_pre_process_modifications() + cgmes_modified_model = export_to_cgmes_zip([network_triplets]) + for component in opdm_object['opde:Component']: + # Map exported modified file with initial inside opdm object + cgmes_file = [i for i in cgmes_modified_model if i.name == component['opdm:Profile']['pmd:fileName']][0] + cgmes_file.name = component['opdm:Profile']['pmd:content-reference'] + logger.info(f"Uploading modified model content to Minio: {cgmes_file.name}") + self.minio_service.upload_object(file_path_or_file_object=cgmes_file, + bucket_name=opdm_object['minio-bucket'], + tags={"state": "modified"}) + # Collect both pre and post loadflow validation reports and merge report.update(pre_lf_validation.report) report.update(post_lf_validation.report) + report.update(pre_merge_modification.report) # Include relevant metadata fields report['@scenario_timestamp'] = opdm_object['pmd:scenarioDate'] @@ -199,7 +251,6 @@ def handle(self, message: bytes, properties: dict, **kwargs): try: logger.info("Updating OPDM metadata in Elastic with model valid status") # self.update_opdm_metadata_object(id=opdm_object['opde:Id'], body={'valid': valid}) - opdm_object = clean_data_from_opdm_objects(opdm_objects=[opdm_object])[0] opdm_object["valid"] = valid self.elastic_service.send_to_elastic_bulk( index=METADATA_ELK_INDEX, @@ -217,14 +268,6 @@ def handle(self, message: bytes, properties: dict, **kwargs): except Exception as error: logger.error(f"Validation report sending to Elastic failed: {error}") - # Send metadata to quality module queue - try: - report['opdm_object'] = opdm_object - rabbit.BlockingClient().publish(payload=json.dumps(report), exchange_name=RABBIT_QUALITY_EXCHANGE, - headers={"opde:Object-Type": "IGM"}) - except: - logger.error(f"Sending metadata to RabbitMQ failed: {error}") - logger.info(f"Model validation status: {valid} [duration {report['duration_s']}s]") return message, properties @@ -243,16 +286,10 @@ def handle(self, message: bytes, properties: dict, **kwargs): #logging.getLogger('powsybl').setLevel(1) opdm = OPDM() - - opdm_metadata = json.loads(message) - - # Get network models data from object storage - opdm_objects = [models.get_content(metadata=opdm_object) for opdm_object in opdm_metadata] - - # latest_boundary = opdm.get_latest_boundary() + latest_boundary = opdm.get_latest_boundary() available_models = opdm.get_latest_models_and_download(time_horizon='1D', - scenario_date="2025-05-27T09:30", - tso="AST") + scenario_date="2025-06-16T09:30", + tso="LITGRID") validated_models = [] # Validate models diff --git a/emf/model_validator/validator_functions.py b/emf/model_validator/validator_functions.py index d50f4b67..36302b37 100644 --- a/emf/model_validator/validator_functions.py +++ b/emf/model_validator/validator_functions.py @@ -1,7 +1,7 @@ import logging import pandas import triplets -from emf.common.loadflow_tool.helper import get_opdm_data_from_models +from emf.common.loadflow_tool.helper import get_opdm_data_from_models, create_opdm_objects, export_to_cgmes_zip logger = logging.getLogger(__name__) @@ -108,3 +108,34 @@ def check_not_retained_switches_between_nodes(original_data, open_not_retained_s original_data = triplets.rdf_parser.update_triplet_from_triplet(original_data, open_switches) return original_data, updated_switches + + +def get_not_retained_switches_between_nodes(original_data): + """ + For the loadflow open all the non-retained switches that connect different topological nodes + Currently it is seen to help around 9 to 10 Kirchhoff 1st law errors from 2 TSOs + :param original_data: original models in triplets format + :return: updated original data + """ + updated_switches = False + original_models = get_opdm_data_from_models(original_data) + not_retained_switches = original_models[(original_models['KEY'] == 'Switch.retained') + & (original_models['VALUE'] == "false")][['ID']] + closed_switches = original_models[(original_models['KEY'] == 'Switch.open') + & (original_models['VALUE'] == 'false')] + not_retained_closed = not_retained_switches.merge(closed_switches[['ID']], on='ID') + terminals = original_models.type_tableview('Terminal').rename_axis('Terminal').reset_index() + terminals = terminals[['Terminal', + # 'ACDCTerminal.connected', + 'Terminal.ConductingEquipment', + 'Terminal.TopologicalNode']] + not_retained_terminals = (terminals.rename(columns={'Terminal.ConductingEquipment': 'ID'}) + .merge(not_retained_closed, on='ID')) + if not_retained_terminals.empty: + return original_data, updated_switches + between_tn = ((not_retained_terminals.groupby('ID')[['Terminal.TopologicalNode']] + .apply(lambda x: check_switch_terminals(x, 'Terminal.TopologicalNode'))) + .reset_index(name='same_TN')) + between_tn = between_tn[between_tn['same_TN']] + + return between_tn diff --git a/emf/model_validator/worker.py b/emf/model_validator/worker.py index 219111ca..80fce5c8 100644 --- a/emf/model_validator/worker.py +++ b/emf/model_validator/worker.py @@ -19,6 +19,7 @@ # RabbitMQ consumer implementation consumer = rabbit.RMQConsumer(queue=INPUT_RMQ_QUEUE, message_handlers=[HandlerModelsValidator()], + forward=OUTPUT_RMQ_EXCHANGE, ) try: From 543bb2b61a24700183e669f048601e524cee98c8 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Tue, 17 Jun 2025 13:50:29 +0300 Subject: [PATCH 19/67] aligned quality module index names --- config/model_quality/model_quality.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/model_quality/model_quality.properties b/config/model_quality/model_quality.properties index 5dca1634..e6468540 100644 --- a/config/model_quality/model_quality.properties +++ b/config/model_quality/model_quality.properties @@ -1,4 +1,4 @@ [MAIN] INPUT_RMQ_QUEUE = model-quality.test -ELK_QUALITY_INDEX = test-emf-model-quality -ELK_STATISTICS_INDEX = test-emf-model-statistics \ No newline at end of file +ELK_QUALITY_INDEX = emfos-model-quality +ELK_STATISTICS_INDEX = emfos-model-statistics \ No newline at end of file From 2b58afd361c8bbf8ba6d9b23f13e8a5c9bae05e8 Mon Sep 17 00:00:00 2001 From: Simon <69009411+snec-afk@users.noreply.github.com> Date: Tue, 17 Jun 2025 15:38:22 +0300 Subject: [PATCH 20/67] Update publish-workers.yml --- .github/workflows/publish-workers.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-workers.yml b/.github/workflows/publish-workers.yml index fb5649ed..d559993e 100644 --- a/.github/workflows/publish-workers.yml +++ b/.github/workflows/publish-workers.yml @@ -5,8 +5,8 @@ on: push: tags: - "v*" - branches: - - main + # branches: + # - main workflow_dispatch: jobs: From 8368b7847a6b2da2175e78a18a116c9ef218f20b Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Thu, 19 Jun 2025 11:51:06 +0300 Subject: [PATCH 21/67] refactoring ongoing to optimize post-processing --- config/cgm_worker/merger.properties | 5 +- config/cgm_worker/post_processing.properties | 5 + docker/Dockerfile | 1 - emf/common/helpers/cgmes.py | 53 + emf/common/helpers/loadflow.py | 207 ++++ emf/common/helpers/opdm_objects.py | 220 +++++ emf/common/helpers/tasks.py | 38 + .../{time_helper.py => helpers/time.py} | 2 + emf/common/helpers/utils.py | 107 ++ emf/common/integrations/minio_api.py | 12 +- emf/common/loadflow_tool/helper.py | 653 ------------- emf/model_merger/merge_functions.py | 925 +++++++++--------- emf/model_merger/model_merger.py | 145 +-- emf/model_merger/scaler.py | 4 +- emf/model_merger/temporary.py | 400 ++++++++ emf/model_merger/temporary_fixes.py | 462 --------- emf/model_retriever/model_retriever.py | 6 +- emf/model_validator/model_validator.py | 13 +- emf/model_validator/validator_functions.py | 8 +- emf/task_generator/manual_worker.py | 3 +- emf/task_generator/task_generator.py | 72 +- emf/task_generator/worker.py | 3 +- 22 files changed, 1600 insertions(+), 1744 deletions(-) create mode 100644 config/cgm_worker/post_processing.properties create mode 100644 emf/common/helpers/cgmes.py create mode 100644 emf/common/helpers/loadflow.py create mode 100644 emf/common/helpers/opdm_objects.py create mode 100644 emf/common/helpers/tasks.py rename emf/common/{time_helper.py => helpers/time.py} (99%) create mode 100644 emf/common/helpers/utils.py delete mode 100644 emf/common/loadflow_tool/helper.py create mode 100644 emf/model_merger/temporary.py delete mode 100644 emf/model_merger/temporary_fixes.py diff --git a/config/cgm_worker/merger.properties b/config/cgm_worker/merger.properties index 270acc26..87115bf7 100644 --- a/config/cgm_worker/merger.properties +++ b/config/cgm_worker/merger.properties @@ -7,7 +7,6 @@ OUTPUT_MINIO_BUCKET = opde-confidential-models OUTPUT_MINIO_FOLDER = EMFOS/RMM MERGE_REPORT_ELK_INDEX = emfos-merge-reports OPDE_MODELS_ELK_INDEX = emfos-opde-models -SMALL_ISLAND_SIZE = 10 MERGE_LOAD_FLOW_SETTINGS = CGM_RELAXED_3_CUSTOM -NET_INTERCHANGE_THRESHOLD = 200 -OPEN_NON_RETAINED_SWITCHES_BETWEEN_TN = True \ No newline at end of file +OPEN_NON_RETAINED_SWITCHES_BETWEEN_TN = True +REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION = True \ No newline at end of file diff --git a/config/cgm_worker/post_processing.properties b/config/cgm_worker/post_processing.properties new file mode 100644 index 00000000..2b029c65 --- /dev/null +++ b/config/cgm_worker/post_processing.properties @@ -0,0 +1,5 @@ +[MAIN] +NET_INTERCHANGE_THRESHOLD = 200 +FIX_INJECTION_ERRORS = True +INJECTION_THRESHOLD = 0.1 +SMALL_ISLAND_SIZE = 10 \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index a2402610..d88a6ebd 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -73,7 +73,6 @@ WORKDIR /src # Copy files to container COPY emf/model_merger/ ./emf/model_merger/ COPY emf/common/ ./emf/common/ -COPY emf/task_generator/ ./emf/task_generator/ COPY config/ ./config/ # Start process diff --git a/emf/common/helpers/cgmes.py b/emf/common/helpers/cgmes.py new file mode 100644 index 00000000..527abaed --- /dev/null +++ b/emf/common/helpers/cgmes.py @@ -0,0 +1,53 @@ +import logging +import json +import pandas as pd +import config +from lxml import etree + +logger = logging.getLogger(__name__) + + +def export_to_cgmes_zip(triplets: list): + namespace_map = { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "cim": "http://iec.ch/TC57/2013/CIM-schema-cim16#", + "md": "http://iec.ch/TC57/61970-552/ModelDescription/1#", + "entsoe": "http://entsoe.eu/CIM/SchemaExtension/3/1#", + } + + rdf_map = json.load(config.paths.cgm_worker.CGMES_v2_4_15_2014_08_07) + + return pd.concat(triplets, ignore_index=True).export_to_cimxml(rdf_map=rdf_map, + namespace_map=namespace_map, + export_undefined=False, + export_type="xml_per_instance_zip_per_xml", + debug=False, + export_to_memory=True) + + +def get_metadata_from_rdfxml(parsed_xml: etree._ElementTree): + """Parse model metadata form xml, returns a dictionary with metadata""" + + assert isinstance(parsed_xml, etree._ElementTree) + + rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" + + header = parsed_xml.find("{*}FullModel") # Version update proof, as long as element name remains the same + meta_elements = header.getchildren() + + # Add model ID from FullModel@about + metadata = {"Model.mRID": header.attrib.get(f'{{{rdf}}}about').split(":")[-1]} + + # Add all other metadata + for element in meta_elements: + key = element.tag.split("}")[1] + value = element.text if element.text else element.attrib.get(f"{{{rdf}}}resource") + if existing_value := metadata.get(key): + value = existing_value + [value] if isinstance(existing_value, list) else [existing_value, value] + metadata[key] = value + + return metadata + + +if __name__ == "__main__": + pass \ No newline at end of file diff --git a/emf/common/helpers/loadflow.py b/emf/common/helpers/loadflow.py new file mode 100644 index 00000000..314505ca --- /dev/null +++ b/emf/common/helpers/loadflow.py @@ -0,0 +1,207 @@ +import logging +import uuid +import re +import pandas as pd +from io import BytesIO +from zipfile import ZipFile, ZIP_DEFLATED +import pypowsybl +from typing import List + +logger = logging.getLogger(__name__) + + +def package_for_pypowsybl(opdm_objects, return_zip: bool = False): + """ + Method to transform OPDM objects into sufficient format binary buffer or zip package + :param opdm_objects: list of OPDM objects + :param return_zip: flag to save OPDM objects as zip package in local directory + :return: binary buffer or zip package file name + """ + output_object = BytesIO() + if return_zip: + output_object = f"{uuid.uuid4()}.zip" + logger.info(f"Adding files to {output_object}") + + with ZipFile(output_object, "w") as global_zip: + for opdm_components in opdm_objects: + for instance in opdm_components['opde:Component']: + with ZipFile(BytesIO(instance['opdm:Profile']['DATA'])) as instance_zip: + for file_name in instance_zip.namelist(): + logger.info(f"Adding file: {file_name}") + global_zip.writestr(file_name, instance_zip.open(file_name).read()) + + return output_object + + +def load_network_model(opdm_objects: List[dict], parameters: dict = None, skip_default_parameters: bool = False): + """ + Loads given list of models (opdm_objects) into pypowsybl using internal (known good) default_parameters + Additional parameters can be specified as a dict in field parameters which will overwrite the default ones if keys + are matching + :param opdm_objects: list of dictionaries following the opdm model format + :param parameters: dictionary of desired parameters for loading models to pypowsybl + :param skip_default_parameters: skip the default parameters + """ + default_parameters = {"iidm.import.cgmes.import-node-breaker-as-bus-breaker": 'true'} + if not skip_default_parameters: + if not parameters: + parameters = default_parameters + else: + # Give a priority to parameters given from outside + parameters = {**default_parameters, **parameters} + + import_report = pypowsybl.report.Reporter() + network = pypowsybl.network.load_from_binary_buffer( + buffer=package_for_pypowsybl(opdm_objects), + reporter=import_report, + parameters=parameters + # parameters={ + # "iidm.import.cgmes.store-cgmes-model-as-network-extension": 'true', + # "iidm.import.cgmes.create-active-power-control-extension": 'true', + # "iidm.import.cgmes.post-processors": ["EntsoeCategory"]} + ) + + logger.info(f"Loaded: {network}") + logger.debug(f"{import_report}") + + return network + + +def get_network_elements(network: pypowsybl.network, + element_type: pypowsybl.network.ElementType, + all_attributes: bool = True, + attributes: List[str] = None, + **kwargs + ): + + _voltage_levels = network.get_voltage_levels(all_attributes=True).rename(columns={"name": "voltage_level_name"}) + _substations = network.get_substations(all_attributes=True).rename(columns={"name": "substation_name"}) + + elements = network.get_elements(element_type=element_type, all_attributes=all_attributes, attributes=attributes, **kwargs) + elements = elements.merge(_voltage_levels, left_on='voltage_level_id', right_index=True, suffixes=(None, '_voltage_level')) + elements = elements.merge(_substations, left_on='substation_id', right_index=True, suffixes=(None, '_substation')) + + # Need to ensure that column 'isHvdc' is present if DANGLING_LINE type is requested + if element_type is pypowsybl.network.ElementType.DANGLING_LINE: + if 'isHvdc' not in elements.columns: + elements['isHvdc'] = '' + + return elements + + +def get_slack_generators(network: pypowsybl.network): + slack_terminals = network.get_extension('slackTerminal') + slack_generators = get_network_elements(network=network, + element_type=pypowsybl.network.ElementType.GENERATOR, + all_attributes=True, + id=slack_terminals['element_id']) + + return slack_generators + + +def get_connected_components_data(network: pypowsybl.network, + bus_count_threshold: int | None = None, + country_col_name: str = 'country'): + buses = get_network_elements(network, pypowsybl.network.ElementType.BUS) + data = buses.groupby('connected_component').agg(countries=(country_col_name, lambda x: list(x.unique())), + bus_count=('name', 'size')) + if bus_count_threshold: + data = data[data.bus_count > bus_count_threshold] + + return data.to_dict('index') + + +def get_model_outages(network: pypowsybl.network): + + # Get network elements + lines = network.get_elements(element_type=pypowsybl.network.ElementType.LINE, all_attributes=True).reset_index(names=['grid_id']) + _voltage_levels = network.get_voltage_levels(all_attributes=True).rename(columns={"name": "voltage_level_name"}) + _substations = network.get_substations(all_attributes=True).rename(columns={"name": "substation_name"}) + lines = lines.merge(_voltage_levels, left_on='voltage_level1_id', right_index=True, suffixes=(None, '_voltage_level')) + lines = lines.merge(_substations, left_on='substation_id', right_index=True, suffixes=(None, '_substation')) + lines['element_type'] = 'LINE' + + dlines = get_network_elements(network, pypowsybl.network.ElementType.DANGLING_LINE).reset_index(names=['grid_id']) + dlines['element_type'] = 'DANGLING_LINE' + + gens = get_network_elements(network, pypowsybl.network.ElementType.GENERATOR).reset_index(names=['grid_id']) + gens['element_type'] = 'GENERATOR' + + # Filter out disconnected elements + disconnected_lines = lines[(lines['connected1'] == False) | (lines['connected2'] == False)] + disconnected_dlines = dlines[dlines['connected'] == False] + disconnected_gens = gens[gens['connected'] == False] + + # Filter out only 330kv and above + disconnected_elements = pd.concat([disconnected_lines, disconnected_dlines, disconnected_gens]) + disconnected_elements = disconnected_elements[disconnected_elements['nominal_v'] >= 330] + + return disconnected_elements.to_dict('records') + + +def parse_pypowsybl_report(report: str): + lines = report.replace('+', '').splitlines() + all_network_dicts = [] + + current_dict = None + base_indent = None + + for line in lines: + stripped_line = line.strip() + + # Identify "Network info" line and its indentation level + if "Network info" in stripped_line: + if current_dict is not None: + # Save the current dictionary if a new "Network info" block starts + all_network_dicts.append(current_dict) + + current_dict = {} + base_indent = len(line) - len(stripped_line) + continue + + if current_dict is not None: + # Calculate the current line's indentation level relative to "Network info" + current_indent = len(line) - len(line.lstrip()) + + # Check for the specific phrase "Network has x buses and y branches" + match = re.match(r"Network has (\d+) buses and (\d+) branches", stripped_line) + if match: + buses = int(match.group(1)) + branches = int(match.group(2)) + current_dict['buses'] = buses + current_dict['branches'] = branches + + # Process lines with key-value pairs after ':' + elif ':' in stripped_line: + dict_name, key_values = stripped_line.split(':', 1) + dict_name = dict_name.strip() + key_values = key_values.strip() + + # Parse key-value pairs + if '=' in key_values: + current_dict[dict_name] = {} + for pair in key_values.split(','): + key, value = map(str.strip, pair.split('=')) + current_dict[dict_name][key] = value + else: + # Handle plain strings after ':' + current_dict[dict_name] = key_values + + else: + # Stop processing this block if indentation level is not greater than base_indent + if current_indent <= base_indent and current_dict: + all_network_dicts.append(current_dict) + current_dict = None + + # Append the last dictionary if it exists + if current_dict is not None and current_dict: + all_network_dicts.append(current_dict) + + # Filter out empty dicts + result = [n for n in all_network_dicts if n] + + return result + + +if __name__ == "__main__": + pass \ No newline at end of file diff --git a/emf/common/helpers/opdm_objects.py b/emf/common/helpers/opdm_objects.py new file mode 100644 index 00000000..71c080e3 --- /dev/null +++ b/emf/common/helpers/opdm_objects.py @@ -0,0 +1,220 @@ +import logging +from io import BytesIO +import pandas as pd +import triplets +from lxml import etree +from emf.common.helpers.time import parse_datetime +from emf.common.helpers.utils import get_xml_from_zip +from emf.common.helpers.cgmes import get_metadata_from_rdfxml + + +logger = logging.getLogger(__name__) + + +def clean_data_from_opdm_objects(opdm_objects: list) -> list: + for opdm_object in opdm_objects: + for component in opdm_object['opde:Component']: + component['opdm:Profile']['DATA'] = None + + return opdm_objects + + +def save_opdm_objects(opdm_objects: dict) -> list: + """ + Function save OPDM objects on to local filesystem + :param opdm_objects: list of OPDM objects + :return: list of exported files + """ + exported_files = [] + for opdm_components in opdm_objects: + for instance in opdm_components['opde:Component']: + file_name = instance['opdm:Profile']['pmd:fileName'] + logger.info(f'Saving: {file_name}') + with open(file_name, 'wb') as instance_zip: + instance_zip.write(instance['opdm:Profile']['DATA']) + exported_files.append(file_name) + + return exported_files + + +def get_opdm_component_data_bytes(opdm_component: dict): + data = BytesIO(opdm_component['opdm:Profile']['DATA']) + data.name = opdm_component['opdm:Profile']['pmd:fileName'] + return data + + +def load_opdm_objects_to_triplets(opdm_objects: list[dict], profile: str | None = None): + if profile: + return pd.read_RDF([get_opdm_component_data_bytes(instance) for model in opdm_objects for instance in model['opde:Component'] if instance['opdm:Profile']['pmd:cgmesProfile'] == profile]) + return pd.read_RDF([get_opdm_component_data_bytes(instance) for model in opdm_objects for instance in model['opde:Component']]) + + +def get_metadata_from_file_name(file_name: str, meta_separator: str = "_"): + """ + Parse OPDM metadata from a filename string into a dictionary. + + Args: + :param file_name: (str): The filename containing metadata separated by underscores + and a file extension + :param meta_separator: (str): How the elements are separated in the string, usually by "_" + + Returns: + dict: Dictionary containing parsed metadata with OPDM-specific keys. + + Raises: + AssertionError: If file_name is not a string. + ValueError: If metadata parsing fails due to incorrect format. + """ + # Constants for OPDM metadata keys + VALID_FROM = 'pmd:validFrom' + TIME_HORIZON = 'pmd:timeHorizon' + CGMES_PROFILE = 'pmd:cgmesProfile' + VERSION_NUMBER = 'pmd:versionNumber' + MERGING_ENTITY = 'pmd:mergingEntity' + MERGING_AREA = 'pmd:mergingArea' + MODEL_PART = 'pmd:modelPartReference' + TSO = "pmd:TSO" + SOURCING_ACTOR = "pmd:sourcingActor" + + # Constant for file type + FILE_TYPE = 'file_type' + + # Input validation + assert isinstance(file_name, str), "file_name must be a string" + + metadata = {} # Meta container + + # Split filename into name and extension + file_name, metadata[FILE_TYPE] = file_name.split(".") + meta_list = file_name.split(meta_separator) + + # Parse main metadata components + if len(meta_list) == 4: + metadata[VALID_FROM], model_authority, metadata[CGMES_PROFILE], metadata[VERSION_NUMBER] = meta_list + metadata[TIME_HORIZON] = "" + elif len(meta_list) == 5: + metadata[VALID_FROM], metadata[TIME_HORIZON], model_authority, metadata[CGMES_PROFILE], metadata[VERSION_NUMBER] = meta_list + else: + logger.warning(f"Parsing error, number of allowed meta in filename is 4 or 5 separated by {meta_separator}: {file_name}") + return metadata + + # Parse model authority components + model_authority_list = model_authority.split("-") + + if len(model_authority_list) == 1: + metadata[MODEL_PART] = model_authority + elif len(model_authority_list) == 2: + metadata[MERGING_ENTITY], metadata[MERGING_AREA] = model_authority_list + elif len(model_authority_list) == 3: + metadata[MERGING_ENTITY], metadata[MERGING_AREA], metadata[MODEL_PART] = model_authority_list + else: + logger.error(f"Parsing error {model_authority}") + + # Add aliases + if metadata.get(MODEL_PART): + metadata[TSO] = metadata[MODEL_PART] + metadata[SOURCING_ACTOR] = metadata[MODEL_PART] + + return metadata + + +def generate_opdm_object_content_reference_from_filename(file_name: str, opdm_object_type: str = "CGMES"): + """ + Generates the file path based on the given parameters and metadata extracted from the filename. + + Parameters: + file_name (str): The name of the file from which metadata will be extracted. + opdm_object_type (str): The type of OPDM object, defaults to "CGMES". + + Returns: + str: Generated file path using the provided template and extracted metadata. + """ + template = "{opdm_object_type}/{processType}/{modelingEntity}/{scenarioTime:%Y%m%d}/{scenarioTime:%H%M00}/{messageType}/{file_name}" + + meta = {key.split(".")[-1]: value for key, value in get_metadata_from_file_name(file_name).items()} + meta["scenarioTime"] = parse_datetime(meta["scenarioTime"]) + meta["file_name"] = file_name + meta["opdm_object_type"] = opdm_object_type + + return template.format(**meta) + + +def get_opdm_metadata_from_rdfxml(parsed_xml: etree._ElementTree): + opdm_metadata_map = { + "Model.mRID": ["opde:Id", "pmd:modelid", "pmd:fullModel_ID"], + "Model.scenarioTime": ["pmd:scenarioDate"], + "Model.created": ["pmd:creationDate"], + "Model.description": ["pmd:description"], + "Model.version": ["pmd:version"], + "Model.DependentOn": ["opde:DependsOn"], + "Model.profile": ["pmd:modelProfile"], + "Model.modelingAuthoritySet": ["pmd:modelingAuthoritySet"], + + } + + metadata = {} + for key, value in get_metadata_from_rdfxml(parsed_xml).items(): + if opdm_keys := opdm_metadata_map.get(key): + for opdm_key in opdm_keys: + metadata[opdm_key] = value + + return metadata + + +def create_opdm_objects(models: list, metadata: dict | None = None, key_profile: str = "SV") -> list: + """ + Function to create OPDM object like structure in memory + input models, is a nested list [[EQ, SSH, TP, SV], [EQ, SSH, TP, SC]] where upper list will become opdm:Object + and nested will be opdm:Profile + + :return: list of OPDM objects + """ + opdm_objects = [] + + for model in models: + opdm_object = {'opde:Component': []} + + for profile_instance in model: + + # Build profile instance metadata + opdm_profile = get_metadata_from_file_name(profile_instance.name) + opdm_profile.update(get_opdm_metadata_from_rdfxml(get_xml_from_zip(profile_instance))) + opdm_profile['pmd:fileName'] = profile_instance.name + opdm_profile["pmd:content-reference"] = generate_opdm_object_content_reference_from_filename(profile_instance.name) + + # Check if key profile and add to main object metadata + if opdm_profile.get('pmd:cgmesProfile') == key_profile: + opdm_object.update(opdm_profile) + + # Add model type - IGM/CGM + # TODO develop logic to also define as CGM metadata object + opdm_object['opde:Object-Type'] = "IGM" + + # Add DATA + opdm_profile['DATA'] = profile_instance.getvalue() + + # Add component to main object + opdm_object['opde:Component'].append({'opdm:Profile': opdm_profile}) + + # If metadata provided, overwrite existing + if metadata: + opdm_object.update(metadata) + + opdm_objects.append(opdm_object) + + return opdm_objects + + +def filename_from_opdm_metadata(metadata: dict, file_type: str | None = None): + model_authority = f"{metadata['pmd:mergingEntity']}-{metadata['pmd:Area']}" + file_name = f"{metadata['pmd:validFrom']}_{metadata['pmd:timeHorizon']}_{model_authority}_{metadata['pmd:cgmesProfile']}_{metadata['pmd:versionNumber']}" + + if file_type: + file_name = f"{file_name}.{file_type}" + + return file_name + + +if __name__ == "__main__": + # Create OPDM objects + opdm_object = create_opdm_objects([rdfzip_files]) \ No newline at end of file diff --git a/emf/common/helpers/tasks.py b/emf/common/helpers/tasks.py new file mode 100644 index 00000000..f4e44c8c --- /dev/null +++ b/emf/common/helpers/tasks.py @@ -0,0 +1,38 @@ +import logging +from datetime import datetime +import config +from emf.common.integrations.elastic import Elastic +from emf.common.config_parser import parse_app_properties + +logger = logging.getLogger(__name__) +parse_app_properties(globals(), config.paths.task_generator.task_generator) + + +def update_task_status(task: dict, status_text: str, publish: bool = True): + """Update task status + Will update task_update_time + Will update task_status + Will append new status to task_status_trace""" + + logger.info(f"Updating Task status to: {status_text}") + + utc_now = datetime.utcnow().isoformat() + + task["task_update_time"] = utc_now + task["task_status"] = status_text + + task["task_status_trace"].append({ + "status": status_text, + "timestamp": utc_now + }) + + # TODO - better handling if elk is not available, possibly set elk connection timeout really small or refactor the sending to happen via rabbit + if publish: + try: + Elastic.send_to_elastic(index=TASK_ELK_INDEX, json_message=task, id=task.get("@id")) + except Exception as e: + logger.error(f"Task publication to Elastic failed with error: {e}") + + +if __name__ == "__main__": + pass \ No newline at end of file diff --git a/emf/common/time_helper.py b/emf/common/helpers/time.py similarity index 99% rename from emf/common/time_helper.py rename to emf/common/helpers/time.py index fda6f096..81d30397 100644 --- a/emf/common/time_helper.py +++ b/emf/common/helpers/time.py @@ -235,6 +235,7 @@ def convert_to_utc(date_time, default_timezone='Europe/Brussels'): return date_time.astimezone(timezone("UTC")) + def convert_to_timezone(date_time, from_timezone="UTC", to_timezone='Europe/Brussels'): """ Converts a datetime object from timezone to timezone. @@ -259,6 +260,7 @@ def convert_to_timezone(date_time, from_timezone="UTC", to_timezone='Europe/Brus def utcnow(): return datetime.now(timezone("UTC")) + def parse_datetime(iso_string, keep_timezone=True): if keep_timezone: diff --git a/emf/common/helpers/utils.py b/emf/common/helpers/utils.py new file mode 100644 index 00000000..64d82880 --- /dev/null +++ b/emf/common/helpers/utils.py @@ -0,0 +1,107 @@ +import logging +from datetime import datetime +from io import BytesIO +from zipfile import ZipFile, ZIP_DEFLATED +from lxml import etree + +logger = logging.getLogger(__name__) + + +def attr_to_dict(instance: object, sanitize_to_strings: bool = False): + """ + Method to return class variables/attributes as dictionary + pypowsybl._pypowsybl.LimitViolation -> dict + :param instance: class instance + :param sanitize_to_strings: flag to convert attributes to string type + :return: dict + """ + + def convert_value(value): + if isinstance(value, datetime): + return value.isoformat() if sanitize_to_strings else value + elif isinstance(value, list): + return [convert_value(item) for item in value] + elif isinstance(value, dict): + return {k: convert_value(v) for k, v in value.items()} + elif hasattr(value, '__dict__'): + return attr_to_dict(value, sanitize_to_strings) + elif sanitize_to_strings: + return str(value) + return value + + attribs = [attr for attr in dir(instance) if (not callable(getattr(instance, attr)) and not attr.startswith("_"))] + result_dict = {attr_key: convert_value(getattr(instance, attr_key)) for attr_key in attribs} + + return result_dict + + +def flatten_dict(nested_dict: dict, parent_key: str = '', separator: str = '.'): + """ + Flattens a nested dictionary. + + Parameters: + - nested_dict (dict): The dictionary to flatten. + - parent_key (str): The base key string used for recursion. + - separator (str): The separator between parent and child keys. + + Returns: + - dict: A flattened dictionary where nested keys are concatenated into a single string. + """ + items = [] + for k, v in nested_dict.items(): + new_key = f"{parent_key}{separator}{k}" if parent_key else k + if isinstance(v, dict): + items.extend(flatten_dict(v, new_key, separator=separator).items()) + elif isinstance(v, list): + for i, item in enumerate(v): + if isinstance(item, dict): + items.extend(flatten_dict(item, f"{new_key}[{i}]", separator=separator).items()) + else: + items.append((f"{new_key}[{i}]", item)) + else: + items.append((new_key, v)) + return dict(items) + + +def filter_and_flatten_dict(nested_dict: dict, keys: list): + """ + Creates a new flat dictionary from specified keys. + + Parameters: + - nested_dict (dict): The original nested dictionary. + - keys (list): The list of keys to include in the new flat dictionary. + + Returns: + - dict: A new flat dictionary with only the specified keys. + """ + flattened = flatten_dict(nested_dict) + return {key: flattened[key] for key in keys if key in flattened} + + +def zip_xml(xml_file_object: BytesIO): + xml_file_name = xml_file_object.name + xml_file_extension = xml_file_name.split(".")[-1] + zip_file_name = xml_file_name.replace(f".{xml_file_extension}", ".zip") + + zip_file_object = BytesIO() + zip_file_object.name = zip_file_name + + # Create and save ZIP + with ZipFile(zip_file_object, 'w', ZIP_DEFLATED) as zipfile: + zipfile.writestr(xml_file_name, xml_file_object.getvalue()) + + return zip_file_object + + +def get_xml_from_zip(zip_file_path: str): + with ZipFile(zip_file_path, 'r') as zipfile_object: + xml_file_name = zipfile_object.namelist()[0] + file_bytes = zipfile_object.read(xml_file_name) + # Convert bytes to file-like object for parsing + xml_tree_object = etree.parse(BytesIO(file_bytes)) + + return xml_tree_object + + +if __name__ == "__main__": + pass \ No newline at end of file diff --git a/emf/common/integrations/minio_api.py b/emf/common/integrations/minio_api.py index 7fb5ae91..3e01732e 100644 --- a/emf/common/integrations/minio_api.py +++ b/emf/common/integrations/minio_api.py @@ -16,7 +16,7 @@ from datetime import datetime, timedelta from aniso8601 import parse_datetime from emf.common.config_parser import parse_app_properties -from emf.common.loadflow_tool.helper import opdm_metadata_from_filename +from emf.common.helpers.opdm_objects import get_metadata_from_file_name urllib3.disable_warnings() logger = logging.getLogger(__name__) @@ -208,16 +208,16 @@ def query_objects(self, bucket_name: str, metadata: dict = None, prefix: str = N return result_list def get_all_objects_name(self, bucket_name: str, prefix: str = None): - objects = self.client.list_objects(bucket_name=bucket_name, prefix=prefix,recursive=True) + objects = self.client.list_objects(bucket_name=bucket_name, prefix=prefix, recursive=True) list_elements = [] for obj in objects: try: object_name = obj.object_name.split("/")[-1] - #only take models that have metadata in the filename + # Only take models that have metadata in the filename if len(object_name.split('-')) > 3: list_elements.append(object_name) - except: - logger.warning(f"Object name not present") + except Exception as error: + logger.warning(f"Object name not present, exception: {error}") return list_elements @@ -303,7 +303,7 @@ def get_latest_models_and_download(self, "pmd:fileName": file_name, "DATA": source_zip.open(file_name).read()} - metadata.update(opdm_metadata_from_filename(file_name)) + metadata.update(get_metadata_from_file_name(file_name)) opdm_profile = {'opdm:Profile': metadata} opdm_object['opde:Component'].append(opdm_profile) diff --git a/emf/common/loadflow_tool/helper.py b/emf/common/loadflow_tool/helper.py deleted file mode 100644 index c297a124..00000000 --- a/emf/common/loadflow_tool/helper.py +++ /dev/null @@ -1,653 +0,0 @@ -from zipfile import ZipFile, ZIP_DEFLATED -from uuid import uuid4 -from io import BytesIO -from inspect import ismethod -from typing import List -from datetime import datetime -import pandas as pd -import pypowsybl -import json -import logging -import pandas -import os -from lxml import etree -import triplets -import uuid -from aniso8601 import parse_datetime -import re -import config - -logger = logging.getLogger(__name__) - -powsybl_default_export_settings = { - "iidm.export.cgmes.base-name": "", - "iidm.export.cgmes.cim-version": "", # 14, 16, 100 - "iidm.export.cgmes.export-boundary-power-flows": "true", - "iidm.export.cgmes.export-power-flows-for-switches": "true", - "iidm.export.cgmes.naming-strategy": "identity", # identity, cgmes, cgmes-fix-all-invalid-ids - "iidm.export.cgmes.profiles": "EQ,TP,SSH,SV", - "iidm.export.cgmes.boundary-EQ-identifier": "", - "iidm.export.cgmes.boundary-TP-identifier": "", - "iidm.export.cgmes.modeling-authority-set": "powsybl.org" -} - - -# TODO - Add comments and docstring -def package_for_pypowsybl(opdm_objects, return_zip: bool = False): - """ - Method to transform OPDM objects into sufficient format binary buffer or zip package - :param opdm_objects: list of OPDM objects - :param return_zip: flag to save OPDM objects as zip package in local directory - :return: binary buffer or zip package file name - """ - output_object = BytesIO() - if return_zip: - output_object = f"{uuid4()}.zip" - logging.info(f"Adding files to {output_object}") - - with ZipFile(output_object, "w") as global_zip: - for opdm_components in opdm_objects: - for instance in opdm_components['opde:Component']: - with ZipFile(BytesIO(instance['opdm:Profile']['DATA'])) as instance_zip: - for file_name in instance_zip.namelist(): - logging.info(f"Adding file: {file_name}") - global_zip.writestr(file_name, instance_zip.open(file_name).read()) - - return output_object - - -def save_opdm_objects(opdm_objects: list) -> list: - """ - Function save OPDM objects on to local filesystem - :param opdm_objects: list of OPDM objects - :return: list of exported files - """ - exported_files = [] - for opdm_components in opdm_objects: - for instance in opdm_components['opde:Component']: - file_name = instance['opdm:Profile']['pmd:fileName'] - logger.info(f'Saving - {file_name}') - with open(file_name, 'wb') as instance_zip: - instance_zip.write(instance['opdm:Profile']['DATA']) - exported_files.append(file_name) - - return exported_files - - -def clean_data_from_opdm_objects(opdm_objects: list) -> list: - for opdm_object in opdm_objects: - for component in opdm_object['opde:Component']: - component['opdm:Profile']['DATA'] = None - - return opdm_objects - - -def create_opdm_objects(models: list, metadata=None, key_profile="SV") -> list: - """ - Function to create OPDM object like structure in memory - input models, is a nested list [[EQ, SSH, TP, SV], [EQ, SSH, TP, SC]] where upper list will become opdm:Object and nested will be opdm:Profile - - :return: list of OPDM objects - """ - opdm_objects = [] - - for model in models: - opdm_object = {'opde:Component': []} - - for profile_instance in model: - - # Build profile instance metadata - opdm_profile = opdm_metadata_from_filename(profile_instance.name) - opdm_profile.update(opdm_metadata_from_rdfxml(get_xml_from_zip(profile_instance))) - opdm_profile['pmd:fileName'] = profile_instance.name - opdm_profile["pmd:content-reference"] = generate_opdm_content_reference_from_filename(profile_instance.name) - - # Check if key profile and add to main object metadata - if opdm_profile.get('pmd:cgmesProfile') == key_profile: - opdm_object.update(opdm_profile) - - # Add model type - IGM/CGM - # TODO develop logic to also define as CGM metadata object - opdm_object['opde:Object-Type'] = "IGM" - - # Add DATA - opdm_profile['DATA'] = profile_instance.getvalue() - - # Add component to main object - opdm_object['opde:Component'].append({'opdm:Profile': opdm_profile}) - - # If metadata provided, overwrite existing - if metadata: - opdm_object.update(metadata) - - opdm_objects.append(opdm_object) - - return opdm_objects - - -def attr_to_dict(instance: object, sanitize_to_strings: bool = False): - """ - Method to return class variables/attributes as dictionary - Example: LimitViolation(subject_id='e49a61d1-632a-11ec-8166-00505691de36', subject_name='', limit_type=HIGH_VOLTAGE, limit=450.0, limit_name='', acceptable_duration=2147483647, limit_reduction=1.0, value=555.6890952917897, side=ONE) - pypowsybl._pypowsybl.LimitViolation -> dict - :param instance: class instance - :param sanitize_to_strings: flag to convert attributes to string type - :return: dict - """ - - def convert_value(value): - if isinstance(value, datetime): - return value.isoformat() if sanitize_to_strings else value - elif isinstance(value, list): - return [convert_value(item) for item in value] - elif isinstance(value, dict): - return {k: convert_value(v) for k, v in value.items()} - elif hasattr(value, '__dict__'): - return attr_to_dict(value, sanitize_to_strings) - elif sanitize_to_strings: - return str(value) - return value - - attribs = [attr for attr in dir(instance) if (not callable(getattr(instance, attr)) and not attr.startswith("_"))] - result_dict = {attr_key: convert_value(getattr(instance, attr_key)) for attr_key in attribs} - - return result_dict - - -def parse_pypowsybl_report(report: str): - lines = report.replace('+', '').splitlines() - all_network_dicts = [] - - current_dict = None - base_indent = None - - for line in lines: - stripped_line = line.strip() - - # Identify "Network info" line and its indentation level - if "Network info" in stripped_line: - if current_dict is not None: - # Save the current dictionary if a new "Network info" block starts - all_network_dicts.append(current_dict) - - current_dict = {} - base_indent = len(line) - len(stripped_line) - continue - - if current_dict is not None: - # Calculate the current line's indentation level relative to "Network info" - current_indent = len(line) - len(line.lstrip()) - - # Check for the specific phrase "Network has x buses and y branches" - match = re.match(r"Network has (\d+) buses and (\d+) branches", stripped_line) - if match: - buses = int(match.group(1)) - branches = int(match.group(2)) - current_dict['buses'] = buses - current_dict['branches'] = branches - - # Process lines with key-value pairs after ':' - elif ':' in stripped_line: - dict_name, key_values = stripped_line.split(':', 1) - dict_name = dict_name.strip() - key_values = key_values.strip() - - # Parse key-value pairs - if '=' in key_values: - current_dict[dict_name] = {} - for pair in key_values.split(','): - key, value = map(str.strip, pair.split('=')) - current_dict[dict_name][key] = value - else: - # Handle plain strings after ':' - current_dict[dict_name] = key_values - - else: - # Stop processing this block if indentation level is not greater than base_indent - if current_indent <= base_indent and current_dict: - all_network_dicts.append(current_dict) - current_dict = None - - # Append the last dictionary if it exists - if current_dict is not None and current_dict: - all_network_dicts.append(current_dict) - - # Filter out empty dicts - result = [n for n in all_network_dicts if n] - - return result - - -def get_network_elements(network: pypowsybl.network, - element_type: pypowsybl.network.ElementType, - all_attributes: bool = True, - attributes: List[str] = None, - **kwargs - ): - - _voltage_levels = network.get_voltage_levels(all_attributes=True).rename(columns={"name": "voltage_level_name"}) - _substations = network.get_substations(all_attributes=True).rename(columns={"name": "substation_name"}) - - elements = network.get_elements(element_type=element_type, all_attributes=all_attributes, attributes=attributes, **kwargs) - elements = elements.merge(_voltage_levels, left_on='voltage_level_id', right_index=True, suffixes=(None, '_voltage_level')) - elements = elements.merge(_substations, left_on='substation_id', right_index=True, suffixes=(None, '_substation')) - - # Need to ensure that column 'isHvdc' is present if DANGLING_LINE type is requested - if element_type is pypowsybl.network.ElementType.DANGLING_LINE: - if 'isHvdc' not in elements.columns: - elements['isHvdc'] = '' - - return elements - - -def get_slack_generators(network: pypowsybl.network): - slack_terminals = network.get_extension('slackTerminal') - slack_generators = get_network_elements(network=network, - element_type=pypowsybl.network.ElementType.GENERATOR, - all_attributes=True, - id=slack_terminals['element_id']) - - return slack_generators - - -def get_connected_components_data(network: pypowsybl.network, - bus_count_threshold: int | None = None, - country_col_name: str = 'country'): - buses = get_network_elements(network, pypowsybl.network.ElementType.BUS) - data = buses.groupby('connected_component').agg(countries=(country_col_name, lambda x: list(x.unique())), - bus_count=('name', 'size')) - if bus_count_threshold: - data = data[data.bus_count > bus_count_threshold] - - return data.to_dict('index') - - -def load_model(opdm_objects: List[dict], parameters: dict = None, skip_default_parameters: bool = False): - """ - Loads given list of models (opdm_objects) into pypowsybl using internal (known good) default_parameters - Additional parameters can be specified as a dict in field parameters which will overwrite the default ones if keys - are matching - :param opdm_objects: list of dictionaries following the opdm model format - :param parameters: dictionary of desired parameters for loading models to pypowsybl - :param skip_default_parameters: skip the default parameters - """ - default_parameters = {"iidm.import.cgmes.import-node-breaker-as-bus-breaker": 'true'} - if not skip_default_parameters: - if not parameters: - parameters = default_parameters - else: - # Give a priority to parameters given from outside - parameters = {**default_parameters, **parameters} - - import_report = pypowsybl.report.Reporter() - network = pypowsybl.network.load_from_binary_buffer( - buffer=package_for_pypowsybl(opdm_objects), - reporter=import_report, - parameters=parameters - # parameters={ - # "iidm.import.cgmes.store-cgmes-model-as-network-extension": 'true', - # "iidm.import.cgmes.create-active-power-control-extension": 'true', - # "iidm.import.cgmes.post-processors": ["EntsoeCategory"]} - ) - - logger.info(f"Loaded: {network}") - logger.debug(f"{import_report}") - - return network - - -def opdmprofile_to_bytes(opdm_profile): - data = BytesIO(opdm_profile['opdm:Profile']['DATA']) - data.name = opdm_profile['opdm:Profile']['pmd:fileName'] - return data - - -def load_opdm_data(opdm_objects, profile=None): - if profile: - return pandas.read_RDF([opdmprofile_to_bytes(instance) for model in opdm_objects for instance in model['opde:Component'] if instance['opdm:Profile']['pmd:cgmesProfile'] == profile]) - return pandas.read_RDF([opdmprofile_to_bytes(instance) for model in opdm_objects for instance in model['opde:Component']]) - - -def get_opdm_data_from_models(model_data: list | pandas.DataFrame): - """ - Check if input is already parsed to triplets. Do it otherwise - :param model_data: input models - :return triplets - """ - if not isinstance(model_data, pandas.DataFrame): - model_data = load_opdm_data(model_data) - return model_data - - -def filename_from_opdm_metadata(metadata): - model_part = metadata.get('pmd:modelPartReference', None) - - if model_part: - model_authority = f"{metadata['pmd:mergingEntity']}-{metadata['pmd:mergingArea']}-{model_part}" - - else: - model_authority = f"{metadata['pmd:mergingEntity']}-{metadata['pmd:mergingArea']}" - - file_name = f"{metadata['pmd:validFrom']}_{metadata['pmd:timeHorizon']}_{model_authority}_{metadata['pmd:cgmesProfile']}_{metadata['pmd:versionNumber']}" - file_name = ".".join([file_name, metadata["file_type"]]) - - return file_name - - -def opdm_metadata_from_filename(file_name: str, meta_separator: str = "_"): - """ - Parse OPDM metadata from a filename string into a dictionary. - - Args: - :param file_name: (str): The filename containing metadata separated by underscores - and a file extension - :param meta_separator: (str): How the elements are seperated in the string, usually by "_" - - Returns: - dict: Dictionary containing parsed metadata with OPDM-specific keys. - - Raises: - AssertionError: If file_name is not a string. - ValueError: If metadata parsing fails due to incorrect format. - """ - # Constants for opdm metadata keys - VALID_FROM = 'pmd:validFrom' - TIME_HORIZON = 'pmd:timeHorizon' - CGMES_PROFILE = 'pmd:cgmesProfile' - VERSION_NUMBER = 'pmd:versionNumber' - MERGING_ENTITY = 'pmd:mergingEntity' - MERGING_AREA = 'pmd:mergingArea' - MODEL_PART = 'pmd:modelPartReference' - TSO = "pmd:TSO" - SOURCING_ACTOR = "pmd:sourcingActor" - - # Constant for file type - FILE_TYPE = 'file_type' - - # Input validation - assert isinstance(file_name, str), "file_name must be a string" - - metadata = {} # Meta container - - # Split filename into name and extension - file_name, metadata[FILE_TYPE] = file_name.split(".") - meta_list = file_name.split(meta_separator) - - # Parse main metadata components - if len(meta_list) == 4: - metadata[VALID_FROM], model_authority, metadata[CGMES_PROFILE], metadata[VERSION_NUMBER] = meta_list - metadata[TIME_HORIZON] = "" - elif len(meta_list) == 5: - metadata[VALID_FROM], metadata[TIME_HORIZON], model_authority, metadata[CGMES_PROFILE], metadata[VERSION_NUMBER] = meta_list - else: - logger.warning(f"Parsing error, number of allowed meta in filename is 4 or 5 separated by {meta_separator}: {file_name}") - return metadata - - # Parse model authority components - model_authority_list = model_authority.split("-") - - if len(model_authority_list) == 1: - metadata[MODEL_PART] = model_authority - elif len(model_authority_list) == 2: - metadata[MERGING_ENTITY], metadata[MERGING_AREA] = model_authority_list - elif len(model_authority_list) == 3: - metadata[MERGING_ENTITY], metadata[MERGING_AREA], metadata[MODEL_PART] = model_authority_list - else: - logger.error(f"Parsing error {model_authority}") - - # Add aliases - if metadata.get(MODEL_PART): - metadata[TSO] = metadata[MODEL_PART] - metadata[SOURCING_ACTOR] = metadata[MODEL_PART] - - return metadata - - -def metadata_from_rdfxml(parsed_xml: etree._ElementTree): - """Parse model metadata form xml, returns a dictionary with metadata""" - - assert isinstance(parsed_xml, etree._ElementTree) - - rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" - - header = parsed_xml.find("{*}FullModel") # Version update proof, as long as element name remains the same - meta_elements = header.getchildren() - - # Add model ID from FullModel@about - metadata = {"Model.mRID": header.attrib.get(f'{{{rdf}}}about').split(":")[-1]} - - # Add all other metadata - for element in meta_elements: - - key = element.tag.split("}")[1] - - value = element.text if element.text else element.attrib.get(f"{{{rdf}}}resource") - - if existing_value := metadata.get(key): - value = existing_value + [value] if isinstance(existing_value, list) else [existing_value, value] - - metadata[key] = value - - return metadata - - -def opdm_metadata_from_rdfxml(parsed_xml: etree._ElementTree): - opdm_metadata_map = { - - "Model.mRID": ["opde:Id", "pmd:modelid", "pmd:fullModel_ID"], - "Model.scenarioTime": ["pmd:scenarioDate"], - "Model.created": ["pmd:creationDate"], - "Model.description": ["pmd:description"], - "Model.version": ["pmd:version"], - "Model.DependentOn": ["opde:DependsOn"], - "Model.profile": ["pmd:modelProfile"], - "Model.modelingAuthoritySet": ["pmd:modelingAuthoritySet"], - - } - - metadata = {} - - for key, value in metadata_from_rdfxml(parsed_xml).items(): - if opdm_keys := opdm_metadata_map.get(key): - for opdm_key in opdm_keys: - metadata[opdm_key] = value - - return metadata - - -# TODO -# "pmd:fileName": "20250317T2230Z_14_D7_SSH_011.zip", -# "pmd:content-reference": "CGMES/14/D7/20250317/223000/SSH/20250317T2230Z_14_D7_SSH_011.zip", -# "pmd:Content-Type": "CGMES", - - -def get_xml_from_zip(zip_file_path): - with ZipFile(zip_file_path, 'r') as zipfile_object: - xml_file_name = zipfile_object.namelist()[0] - file_bytes = zipfile_object.read(xml_file_name) - # Convert bytes to file-like object for parsing - xml_tree_object = etree.parse(BytesIO(file_bytes)) - - return xml_tree_object - - -def zip_xml(xml_file_object): - xml_file_name = xml_file_object.name - xml_file_extension = xml_file_name.split(".")[-1] - zip_file_name = xml_file_name.replace(f".{xml_file_extension}", ".zip") - - zip_file_object = BytesIO() - zip_file_object.name = zip_file_name - - # Create and save ZIP - with ZipFile(zip_file_object, 'w', ZIP_DEFLATED) as zipfile: - zipfile.writestr(xml_file_name, xml_file_object.getvalue()) - - return zip_file_object - - -def get_metadata_from_filename(file_name): - # Separators - file_type_separator = "." - meta_separator = "_" - entity_and_domain_separator = "-" - - logger.debug(file_name) - file_metadata = {} - file_name, file_type = file_name.split(file_type_separator) - - # Parse file metadata - file_meta_list = file_name.split(meta_separator) - - # Naming before QoDC 2.1, where EQ might not have processType - if len(file_meta_list) == 4: - - file_metadata["Model.scenarioTime"], \ - file_metadata["Model.modelingEntity"], \ - file_metadata["Model.messageType"], \ - file_metadata["Model.version"] = file_meta_list - file_metadata["Model.processType"] = "" - - logger.warning("Only 4 meta elements found, expecting 5, setting Model.processType to empty string") - - # Naming after QoDC 2.1, always 5 positions - elif len(file_meta_list) == 5: - - file_metadata["Model.scenarioTime"], \ - file_metadata["Model.processType"], \ - file_metadata["Model.modelingEntity"], \ - file_metadata["Model.messageType"], \ - file_metadata["Model.version"] = file_meta_list - - else: - logger.error("Non CGMES file {}".format(file_name)) - - if file_metadata.get("Model.modelingEntity"): - - entity_and_area_list = file_metadata["Model.modelingEntity"].split(entity_and_domain_separator) - - if len(entity_and_area_list) == 1: - file_metadata["Model.mergingEntity"], \ - file_metadata["Model.domain"] = "", "" # Set empty string for both - file_metadata["Model.forEntity"] = entity_and_area_list[0] - - if len(entity_and_area_list) == 2: - file_metadata["Model.mergingEntity"], \ - file_metadata["Model.domain"] = entity_and_area_list - file_metadata["Model.forEntity"] = "" - - if len(entity_and_area_list) == 3: - file_metadata["Model.mergingEntity"], \ - file_metadata["Model.domain"], \ - file_metadata["Model.forEntity"] = entity_and_area_list - - return file_metadata - - -def generate_opdm_content_reference_from_filename(file_name: str, opdm_object_type="CGMES"): - """ - Generates the file path based on the given parameters and metadata extracted from the filename. - - Parameters: - file_name (str): The name of the file from which metadata will be extracted. - opdm_object_type (str): The type of OPDM object, defaults to "CGMES". - - Returns: - str: Generated file path using the provided template and extracted metadata. - - """ - template = "{opdm_object_type}/{processType}/{modelingEntity}/{scenarioTime:%Y%m%d}/{scenarioTime:%H%M00}/{messageType}/{file_name}" - - meta = {key.split(".")[-1]: value for key, value in get_metadata_from_filename(file_name).items()} - meta["scenarioTime"] = parse_datetime(meta["scenarioTime"]) - meta["file_name"] = file_name - meta["opdm_object_type"] = opdm_object_type - - return template.format(**meta) - - -def export_model(network: pypowsybl.network, opdm_object_meta, profiles=None): - - if profiles: - profiles = ",".join([str(profile) for profile in profiles]) - else: - profiles = "SV,SSH,TP,EQ" - - file_base_name = filename_from_opdm_metadata(opdm_object_meta).split(".xml")[0] - - bytes_object = network.save_to_binary_buffer( - format="CGMES", - parameters={ - "iidm.export.cgmes.modeling-authority-set": opdm_object_meta['pmd:modelingAuthoritySet'], - "iidm.export.cgmes.base-name": file_base_name, - "iidm.export.cgmes.profiles": profiles, - # "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids", # identity, cgmes, cgmes-fix-all-invalid-ids - "iidm.export.cgmes.export-sv-injections-for-slacks": "False", - }) - - bytes_object.name = f"{file_base_name}_{uuid.uuid4()}.zip" - - return bytes_object - - -def export_to_cgmes_zip(triplets: list): - namespace_map = { - "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", - "cim": "http://iec.ch/TC57/2013/CIM-schema-cim16#", - "md": "http://iec.ch/TC57/61970-552/ModelDescription/1#", - "entsoe": "http://entsoe.eu/CIM/SchemaExtension/3/1#", - } - - # with open('../../config/cgm_worker/CGMES_v2_4_15_2014_08_07.json', 'r') as file_object: - rdf_map = json.load(config.paths.cgm_worker.CGMES_v2_4_15_2014_08_07) - - return pandas.concat(triplets, ignore_index=True).export_to_cimxml(rdf_map=rdf_map, - namespace_map=namespace_map, - export_undefined=False, - export_type="xml_per_instance_zip_per_xml", - debug=False, - export_to_memory=True) - - -def get_model_outages(network: pypowsybl.network): - - # Get network elements - lines = network.get_elements(element_type=pypowsybl.network.ElementType.LINE, all_attributes=True).reset_index(names=['grid_id']) - _voltage_levels = network.get_voltage_levels(all_attributes=True).rename(columns={"name": "voltage_level_name"}) - _substations = network.get_substations(all_attributes=True).rename(columns={"name": "substation_name"}) - lines = lines.merge(_voltage_levels, left_on='voltage_level1_id', right_index=True, suffixes=(None, '_voltage_level')) - lines = lines.merge(_substations, left_on='substation_id', right_index=True, suffixes=(None, '_substation')) - lines['element_type'] = 'LINE' - - dlines = get_network_elements(network, pypowsybl.network.ElementType.DANGLING_LINE).reset_index(names=['grid_id']) - dlines['element_type'] = 'DANGLING_LINE' - - gens = get_network_elements(network, pypowsybl.network.ElementType.GENERATOR).reset_index(names=['grid_id']) - gens['element_type'] = 'GENERATOR' - - # Filter out disconnected elements - disconnected_lines = lines[(lines['connected1'] == False) | (lines['connected2'] == False)] - disconnected_dlines = dlines[dlines['connected'] == False] - disconnected_gens = gens[gens['connected'] == False] - - # Filter out only 330kv and above - disconnected_elements = pd.concat([disconnected_lines, disconnected_dlines, disconnected_gens]) - disconnected_elements = disconnected_elements[disconnected_elements['nominal_v'] >= 330] - - return disconnected_elements.to_dict('records') - - -if __name__ == "__main__": - - example_file_path = r"C:\Users\kristjan.vilgo\OneDrive - Elering AS\Documents\GitHub\igm_publication\20250314T2330Z_1D_ELERING_012.zip" - - # Find all XML regardless the zip folder depth - rdfxml_files = triplets.rdf_parser.find_all_xml([example_file_path]) - - # Repackage to form zip(xml) - rdfzip_files = [] - for xml in rdfxml_files: - rdfzip_files.append(zip_xml(xml)) - - # Create OPDM objects - opdm_object = create_opdm_objects([rdfzip_files]) diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index cdb68042..fbf0fa9e 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -1,26 +1,27 @@ import zipfile import math from io import BytesIO +import pandas as pd import pypowsybl import logging -import json import sys -from decimal import Decimal import datetime import triplets -import pandas import uuid from uuid import uuid4 import config -from emf.common.loadflow_tool.helper import load_model, load_opdm_data, filename_from_opdm_metadata, attr_to_dict, \ - export_model, get_network_elements, get_opdm_data_from_models +from emf.common.config_parser import parse_app_properties +from emf.common.integrations import elastic from emf.common.loadflow_tool import loadflow_settings -from aniso8601 import parse_datetime -from emf.common.time_helper import parse_datetime as helper_parse_datetime +from emf.model_merger import temporary +from emf.common.helpers.time import parse_datetime +from emf.common.helpers.loadflow import get_model_outages, get_network_elements +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, filename_from_opdm_metadata logger = logging.getLogger(__name__) -SV_INJECTION_LIMIT = 0.1 + +parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.post_processing) def is_valid_uuid(uuid_value): @@ -36,57 +37,72 @@ def is_valid_uuid(uuid_value): except ValueError: return False -def run_lf(merged_model, loadflow_settings=loadflow_settings.CGM_DEFAULT): - - # report = pypowsybl.report.Reporter() - result = pypowsybl.loadflow.run_ac(network=merged_model.network, - parameters=loadflow_settings, - # reporter=loadflow_report, - ) - - result_dict = [attr_to_dict(island) for island in result] - # Modify all nested objects to native data types - for island in result_dict: - island['status'] = island['status'].name - # Extract only first slack bus from internal pypowsybl object - slack_bus_results = island.pop('slack_bus_results') - if slack_bus_results: - island['slack_bus_id'] = getattr(slack_bus_results[0], 'id', 'undefined') - island['active_power_mismatch'] = getattr(slack_bus_results[0], 'active_power_mismatch', float()) - else: - island['slack_bus_id'] = 'undefined' - island['active_power_mismatch'] = float() - # merged_model.loadflow = json.loads(loadflow_report.to_json()) - # merged_model.loadflow = str(loadflow_report) - merged_model.loadflow = [island for island in result_dict if island['reference_bus_id']] - merged_model.loadflow_status = result[0].status.name # store main island loadflow status +def export_merged_model(network: pypowsybl.network, + opdm_object_meta: dict, + profiles: list[str] | None = None, + cgm_convention: bool = True, + ): - return merged_model + # Define which profiles to export + if profiles: + profiles = ",".join(profiles) + else: + profiles = "SV,SSH,TP,EQ" + + # Define whether export using CGM official structure + cgm_export_flag = "False" + if cgm_convention: + cgm_export_flag = "True" + + # Define base name for exported files + file_base_name = filename_from_opdm_metadata(metadata=opdm_object_meta) + + # Define CGMES export parameters + parameters = { + "iidm.export.cgmes.modeling-authority-set": opdm_object_meta['pmd:modelingAuthoritySet'], + "iidm.export.cgmes.base-name": file_base_name, + "iidm.export.cgmes.profiles": profiles, + "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids", # identity, cgmes, cgmes-fix-all-invalid-ids + "iidm.export.cgmes.export-sv-injections-for-slacks": "False", + "iidm.export.cgmes.export-boundary-power-flows": "False", + "iidm.export.cgmes.cgm_export": cgm_export_flag, + } + + # Export to bytes object + bytes_object = network.save_to_binary_buffer(format="CGMES", parameters=parameters) + bytes_object.name = f"{file_base_name}_{uuid.uuid4()}.zip" + logger.info(f"Exported merged model to {bytes_object.name}") + + # TODO set correct naming of exported files + # a = triplets.rdf_parser.find_all_xml([bytes_object]) + return bytes_object -def create_opdm_object_meta(object_id, - time_horizon, - merging_entity, - merging_area, - scenario_date, - mas, - version, - profile, - content_type="CGMES", - file_type="xml" - ): + +def create_merged_model_opdm_object(object_id: str, + time_horizon: str, + merging_entity: str, + merging_area: str, + scenario_date: datetime.datetime, + mas: str, + version: str = "001", + profile: str = "SV", + content_type: str = "CGMES", + ): opdm_object_meta = { + 'opde:Object-Type': 'CGM', 'pmd:fullModel_ID': object_id, 'pmd:creationDate': f"{datetime.datetime.utcnow():%Y-%m-%dT%H:%M:%S.%fZ}", 'pmd:timeHorizon': time_horizon, 'pmd:cgmesProfile': profile, 'pmd:contentType': content_type, - 'pmd:modelPartReference': '', + 'pmd:modelPartReference': "-".join([merging_entity, merging_area]), 'pmd:mergingEntity': merging_entity, - 'pmd:mergingArea': merging_area, + 'pmd:Area': merging_area, 'pmd:validFrom': f"{parse_datetime(scenario_date):%Y%m%dT%H%MZ}", 'pmd:modelingAuthoritySet': mas, + "pmd:isFullModel": "true", 'pmd:scenarioDate': f"{parse_datetime(scenario_date):%Y-%m-%dT%H:%M:00Z}", 'pmd:modelid': object_id, 'pmd:description': f""" @@ -96,118 +112,53 @@ def create_opdm_object_meta(object_id, Model: Simplification of reality for given need. """, 'pmd:versionNumber': f"{int(version):03d}", - 'file_type': file_type } return opdm_object_meta -def update_cgm_opdm_object_meta(opdm_object_meta, merged_model): - - opdm_object_meta.update({'opde:Object-Type': 'CGM', - "pmd:isFullModel": "true", - "pmd:Area": opdm_object_meta['pmd:mergingArea'], - "pmd:version": opdm_object_meta['pmd:versionNumber'], - "pmd:modelid": opdm_object_meta['pmd:fullModel_ID'], - 'pmd:content-reference': merged_model.content_reference, - "pmd:modelPartReference": opdm_object_meta['pmd:mergingEntity']+'-'+opdm_object_meta['pmd:mergingArea'], - }) - - return opdm_object_meta - - -def update_FullModel_from_OpdmObject(data, opdm_object): +def update_header_from_opdm_object(data: pd.DataFrame, opdm_object: dict): return triplets.cgmes_tools.update_FullModel_from_dict(data, metadata={ "Model.version": f"{int(opdm_object['pmd:versionNumber']):03d}", "Model.created": f"{parse_datetime(opdm_object['pmd:creationDate']):%Y-%m-%dT%H:%M:%S.%fZ}", "Model.mergingEntity": opdm_object['pmd:mergingEntity'], - "Model.domain": opdm_object['pmd:mergingArea'], + "Model.domain": opdm_object['pmd:Area'], "Model.scenarioTime": f"{parse_datetime(opdm_object['pmd:scenarioDate']):%Y-%m-%dT%H:%M:00Z}", "Model.description": opdm_object['pmd:description'], - "Model.processType": opdm_object['pmd:timeHorizon'] + "Model.processType": opdm_object['pmd:timeHorizon'], }) -def revert_ids_back(exported_model, triplets_data, revert_ids: bool = True): - """ - As pypowsybl creates its own unique uuids for the cases when the originals do not match the criteria then this - takes the naming_strategy.csv provided by the pypowsybl and reverts those ids back if it is applicable - :param exported_model: binary object from pypowsybl - :param triplets_data: profile(s) converted to triplets - :param revert_ids: True = fix, False=report only - :return (updated) triplets data - """ - - contents = zipfile.ZipFile(exported_model) - naming_strategy = pandas.DataFrame() - - for file_name in contents.namelist(): - if 'naming_strategy' in file_name: - naming_strategy = pandas.read_csv(filepath_or_buffer=BytesIO(contents.read(file_name)), sep=';') - break - if not naming_strategy.empty: - existing_values = triplets_data.merge(naming_strategy, left_on='VALUE', right_on='CgmesUuid') - existing_values = existing_values[existing_values['IidmId'] != 'unknown'] - if not existing_values.empty: - if not revert_ids: - logger.error(f"Found {len(existing_values.index)} changed ids, consider dangling reference errors") - return triplets_data - logger.warning(f"Mapping {len(existing_values.index)} ids back") - existing_values['VALUE'] = existing_values['IidmId'] - new_existing_values = existing_values[['ID', 'KEY', 'VALUE', 'INSTANCE_ID']] - triplets_data = triplets.rdf_parser.update_triplet_from_triplet(triplets_data, new_existing_values) - return triplets_data - - - -def create_sv_and_updated_ssh(merged_model, original_models, models_as_triplets, scenario_date, time_horizon, version, merging_area, merging_entity, mas): +def update_merged_model_sv(sv_data: bytes, opdm_object_meta: dict): + # Load SV profile data + sv_data = pd.read_RDF([sv_data]) - ### SV ### - # Set Metadata - SV_ID = merged_model.network_meta['id'].split("uuid:")[-1] - - opdm_object_meta = create_opdm_object_meta(SV_ID, - time_horizon, - merging_entity, - merging_area, - scenario_date, - mas, - version, - profile="SV") - - exported_model = export_model(merged_model.network, opdm_object_meta, ["SV"]) - logger.info(f"Exporting merged model to {exported_model.name}") - - # Load SV data - sv_data = pandas.read_RDF([exported_model]) - - # Fix naming - sv_data = revert_ids_back(exported_model=exported_model, triplets_data=sv_data) - - # Update - sv_data.set_VALUE_at_KEY(key='label', value=filename_from_opdm_metadata(opdm_object_meta)) - sv_data = triplets.cgmes_tools.update_FullModel_from_filename(sv_data) + # Update rdfxml header from opdm object metadata + sv_data = update_header_from_opdm_object(data=sv_data, opdm_object=opdm_object_meta) - # Update metadata - sv_data = update_FullModel_from_OpdmObject(sv_data, opdm_object_meta) + # Update file name at 'label' key + sv_data.set_VALUE_at_KEY(key='label', value=filename_from_opdm_metadata(opdm_object_meta, file_type="xml")) - # Update filename - sv_data = triplets.cgmes_tools.update_filename_from_FullModel(sv_data) - - # Check and fix SV id + # Check and fix SV id if necessary updated_sv_id_map = {} for old_id in sv_data.query("KEY == 'Type' and VALUE == 'FullModel'").ID.unique(): if not is_valid_uuid(old_id): new_id = str(uuid4()) updated_sv_id_map[old_id] = new_id - logger.info(f"SV id: {old_id} is not valid. Assigning {new_id}") + logger.warning(f"SV id {old_id} is not valid, assigning: {new_id}") sv_data = sv_data.replace(updated_sv_id_map) + return sv_data + + +def create_updated_ssh(merged_model, original_models, models_as_triplets): + # TODO rewrite to use pypowsybl exported SSH + ### SSH ## # Load original SSH data to created updated SSH - ssh_data = load_opdm_data(original_models, "SSH") + ssh_data = load_opdm_objects_to_triplets(opdm_objects=original_models, profile="SSH") ssh_data = triplets.cgmes_tools.update_FullModel_from_filename(ssh_data) # Update SSH Model.scenarioTime @@ -288,12 +239,12 @@ def create_sv_and_updated_ssh(merged_model, original_models, models_as_triplets, sv_data = sv_data.replace(updated_ssh_id_map) # Add SSH supersedes reference to old SSH - ssh_supersedes_data = pandas.DataFrame([{"ID": item[1], "KEY": "Model.Supersedes", "VALUE": item[0]} for item in updated_ssh_id_map.items()]) + ssh_supersedes_data = pd.DataFrame([{"ID": item[1], "KEY": "Model.Supersedes", "VALUE": item[0]} for item in updated_ssh_id_map.items()]) ssh_supersedes_data['INSTANCE_ID'] = ssh_data.query("KEY == 'Type'").merge(ssh_supersedes_data.ID)['INSTANCE_ID'] ssh_data = ssh_data.update_triplet_from_triplet(ssh_supersedes_data) # Update SSH metadata - ssh_data = update_FullModel_from_OpdmObject(ssh_data, opdm_object_meta) + ssh_data = update_header_from_opdm_object(ssh_data, opdm_object_meta) # Update SSH filenames filename_mask = "{scenarioTime:%Y%m%dT%H%MZ}_{processType}_{mergingEntity}-{domain}-{forEntity}_{messageType}_{version:03d}" @@ -302,167 +253,21 @@ def create_sv_and_updated_ssh(merged_model, original_models, models_as_triplets, return sv_data, ssh_data, opdm_object_meta -def fix_sv_shunts(sv_data, models_as_triplets): - """Remove Shunt Sections for EQV Shunts""" - - equiv_shunt = models_as_triplets.query("KEY == 'Type' and VALUE == 'EquivalentShunt'") - if len(equiv_shunt) > 0: - shunts_to_remove = sv_data.merge( - sv_data.query("KEY == 'SvShuntCompensatorSections.ShuntCompensator'").merge(equiv_shunt.ID, left_on='VALUE', - right_on="ID", how='inner', - suffixes=('', '_EQVShunt')).ID) - if len(shunts_to_remove) > 0: - logger.warning(f'Removing invalid SvShuntCompensatorSections for EquivalentShunt') - sv_data = triplets.rdf_parser.remove_triplet_from_triplet(sv_data, shunts_to_remove) - - return sv_data - - -def fix_sv_tapsteps(sv_data, ssh_data): - """Fix SV - Remove Shunt Sections for EQV Shunts""" - - ssh_tap_steps = ssh_data.query("KEY == 'TapChanger.step'") - sv_tap_steps = sv_data.query("KEY == 'SvTapStep.TapChanger'") - - missing_sv_tap_steps = ssh_tap_steps.merge(sv_tap_steps[['VALUE']], left_on='ID', right_on="VALUE", how='left', indicator=True, suffixes=('', '_SV')).query("_merge == 'left_only'") - - tap_steps_to_be_added = [] - SV_INSTANCE_ID = sv_data.INSTANCE_ID.iloc[0] - for tap_changer in missing_sv_tap_steps.itertuples(): - ID = str(uuid4()) - logger.warning( - f'Missing SvTapStep for {tap_changer.ID}, adding SvTapStep {ID} and taking tap value {tap_changer.VALUE} from SSH') - tap_steps_to_be_added.extend([ - (ID, 'Type', 'SvTapStep', SV_INSTANCE_ID), - (ID, 'SvTapStep.TapChanger', tap_changer.ID, SV_INSTANCE_ID), - (ID, 'SvTapStep.position', tap_changer.VALUE, SV_INSTANCE_ID), - ]) - - sv_data = pandas.concat([sv_data, pandas.DataFrame(tap_steps_to_be_added, columns=['ID', 'KEY', 'VALUE', 'INSTANCE_ID'])], ignore_index=True) - - return sv_data - - -def configure_paired_boundarypoint_injections(data): - """Where there are paired boundary points, eqivalent injections need to be modified - Set P and Q to 0 - so that no additional consumption or prduction is on tieline - Set voltage control off - so that no additional consumption or prduction is on tieline - Set terminal to connected - to be sure we have paired connected injections at boundary point - """ - - boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] - #boundary_points = data.type_tableview("ConnectivityNode").reset_index().query("`ConnectivityNode.boundaryPoint` == 'true'") - boundary_points = boundary_points.merge(data.type_tableview("Terminal").reset_index(), left_on="ID", right_on="Terminal.ConnectivityNode", suffixes=('_ConnectivityNode', '_Terminal')) - - injections = data.type_tableview('EquivalentInjection').reset_index().merge(boundary_points, left_on="ID", right_on='Terminal.ConductingEquipment', suffixes=('_ConnectivityNode', '')) - - # Get paired injections at boundary points - paired_injections = injections.groupby("Terminal.ConnectivityNode").filter(lambda x: len(x) == 2) - - # Set terminal status - updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) - updated_terminal_status["KEY"] = "ACDCTerminal.connected" - updated_terminal_status["VALUE"] = "true" - - # Set Regulation off - updated_regulation_status = paired_injections[["ID"]].copy() - updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" - updated_regulation_status["VALUE"] = "false" - - # Set P to 0 - updated_p_value = paired_injections[["ID"]].copy() - updated_p_value["KEY"] = "EquivalentInjection.p" - updated_p_value["VALUE"] = 0 +def ensure_paired_equivalent_injection_compatibility(network: pypowsybl.network): + """Where there are paired boundary points, equivalent injections need to be modified to comply + LEVEL7 rule PairedEICompatibility - # Set Q to 0 - updated_q_value = paired_injections[["ID"]].copy() - updated_q_value["KEY"] = "EquivalentInjection.q" - updated_q_value["VALUE"] = 0 - - return data.update_triplet_from_triplet(pandas.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) - - -def configure_paired_boundarypoint_injections_by_nodes(data): - """Where there are paired boundary points, eqivalent injections need to be modified - Set P and Q to 0 - so that no additional consumption or prduction is on tieline - Set voltage control off - so that no additional consumption or prduction is on tieline - Set terminal to connected - to be sure we have paired connected injections at boundary point - NOTE THAT THIS IS COPY FROM 'configure_paired_boundarypoint_injections' - In some models terminals are missing references to ConnectivityNodes + Set P and Q to 0 - so that no additional consumption or production is on tie line + TODO consider handling if same tie line EQINJ has different ACDCTerminal connected status """ - connectivity_boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] - topological_boundary_points = data.query("KEY == 'TopologicalNode.boundaryPoint' and VALUE == 'true'")[["ID"]] - try: - terminals = data.type_tableview("Terminal").reset_index()[['ID', - 'Terminal.ConductingEquipment', - 'Terminal.ConnectivityNode', - 'Terminal.TopologicalNode']] - except KeyError: - terminals = data.type_tableview("Terminal").reset_index()[['ID', - 'Terminal.ConductingEquipment', - 'Terminal.TopologicalNode']] - injections = data.type_tableview('EquivalentInjection').reset_index()[['ID', - # 'EquivalentInjection.p', - # 'EquivalentInjection.q', - # 'EquivalentInjection.regulationStatus' - ]] - topological_boundary_points = topological_boundary_points.merge(terminals, - left_on="ID", - right_on="Terminal.TopologicalNode", - suffixes=('_TopologicalNode', '_Terminal')) - topological_injections = injections.merge(topological_boundary_points, - left_on="ID", - right_on='Terminal.ConductingEquipment', - suffixes=('_ConnectivityNode', '')) - paired_topological_injections = (topological_injections.groupby("Terminal.TopologicalNode") - .filter(lambda x: len(x) == 2)) - paired_injections = paired_topological_injections - if 'Terminal.ConnectivityNode' in terminals: - connectivity_boundary_points = connectivity_boundary_points.merge(terminals, - left_on="ID", - right_on="Terminal.ConnectivityNode", - suffixes=('_ConnectivityNode', '_Terminal')) - connectivity_injections = injections.merge(connectivity_boundary_points, - left_on="ID", - right_on='Terminal.ConductingEquipment', - suffixes=('_TopologicalNode', '')) - - paired_connectivity_injections = (connectivity_injections.groupby("Terminal.ConnectivityNode") - .filter(lambda x: len(x) == 2)) - merged_injections = paired_connectivity_injections.merge(paired_topological_injections, - on='ID', - how='outer', - indicator=True, - suffixes=('_CN', '_TN')) - only_connectivity_injections = merged_injections[merged_injections['_merge'] == 'left_only'] - only_topological_injections = merged_injections[merged_injections['_merge'] == 'right_only'] - if len(only_connectivity_injections.index) != 0 or len(only_topological_injections.index) == 0: - paired_injections = paired_connectivity_injections - else: - logger.warning(f"Mismatch of finding paired injections from topological nodes and connectivity nodes") - else: - logger.warning(f"Terminals do not contain Connectivity nodes") - # Set terminal status - updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) - updated_terminal_status["KEY"] = "ACDCTerminal.connected" - updated_terminal_status["VALUE"] = "true" + logger.info("Configuring paired boundary points equivalent injections: p0/q0 = 0.0") + dangling_lines = network.get_dangling_lines(all_attributes=True) + paired_dangling_lines = dangling_lines[dangling_lines['paired'] == True] + _updated_p0 = pd.Series(0, index=paired_dangling_lines.index) + _updated_q0 = pd.Series(0, index=paired_dangling_lines.index) + network.update_dangling_lines(id=paired_dangling_lines.index, p0=_updated_p0, q0=_updated_q0) - # Set Regulation off - updated_regulation_status = paired_injections[["ID"]].copy() - updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" - updated_regulation_status["VALUE"] = "false" - - # Set P to 0 - updated_p_value = paired_injections[["ID"]].copy() - updated_p_value["KEY"] = "EquivalentInjection.p" - updated_p_value["VALUE"] = 0 - - # Set Q to 0 - updated_q_value = paired_injections[["ID"]].copy() - updated_q_value["KEY"] = "EquivalentInjection.q" - updated_q_value["VALUE"] = 0 - - return data.update_triplet_from_triplet(pandas.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) + return network def generate_merge_report(merged_model: object, task: dict): @@ -599,194 +404,121 @@ def filter_models(models: list, included_models: list | str = None, excluded_mod return filtered_models -def get_boundary_nodes_between_igms(model_data: list | pandas.DataFrame): - """ - Filters out nodes that are between the igms (mentioned at least 2 igms) - :param model_data: input models - : return series of node ids - """ - model_data = get_opdm_data_from_models(model_data=model_data) - all_boundary_nodes = model_data[(model_data['KEY'] == 'TopologicalNode.boundaryPoint') & - (model_data['VALUE'] == 'true')] - # Get boundary nodes that exist in igms - merged = pandas.merge(all_boundary_nodes, - model_data[(model_data['KEY'] == 'SvVoltage.TopologicalNode')], - left_on='ID', right_on='VALUE', suffixes=('_y', '')) - # Get duplicates (all of them) then duplicated values. keep=False marks all duplicates True, 'first' marks first - # occurrence to false, 'last' marks last occurrence to false. If any of them is used then in case duplicates are 2 - # then 1 is retrieved, if duplicates >3 then duplicates-1 retrieved. So, get all the duplicates and as a second - # step, drop the duplicates - merged = (merged[merged.duplicated(['VALUE'], keep=False)]).drop_duplicates(subset=['VALUE']) - in_several_igms = (merged["VALUE"]).to_frame().rename(columns={'VALUE': 'ID'}) - return in_several_igms - - -def take_best_match_for_sv_voltage(input_data, column_name: str = 'v', to_keep: bool = True): - """ - Returns one row for with sv voltage id for topological node - 1) Take the first - 2) If first is zero take first non-zero row if exists - :param input_data: input dataframe - :param column_name: name of the column - :param to_keep: either to keep or discard a value - """ - first_row = input_data.iloc[0] - if to_keep: - remaining_rows = input_data[input_data[column_name] != 0] - if first_row[column_name] == 0 and not remaining_rows.empty: - first_row = remaining_rows.iloc[0] +def update_model_outages(merged_model: object, tso_list: list, scenario_datetime: str, time_horizon: str): + + area_map = {"LITGRID": "Lithuania", "AST": "Latvia", "ELERING": "Estonia"} + outage_areas = [area_map.get(item, item) for item in tso_list] + + elk_service = elastic.Elastic() + + # Get outage eic-mrid mapping + mrid_map = elk_service.get_docs_by_query(index='config-network*', query={"match_all": {}}, size=10000) + mrid_map['mrid'] = mrid_map['mrid'].str.lstrip('_') + + # Get latest UAP parse date + if time_horizon == 'MO': + merge_type = "Month" + elif time_horizon == 'YR': + merge_type = "Year" else: - remaining_rows = input_data[input_data[column_name] == 0] - if first_row[column_name] != 0 and not remaining_rows.empty: - first_row = remaining_rows.iloc[0] - return first_row + merge_type = "Week" + + body = {"size": 1, "query": {"bool": {"must": [{"match": {"Merge": merge_type}}]}}, + "sort": [{"reportParsedDate": {"order": "desc"}}], "fields": ["reportParsedDate"]} + last_uap_version = elk_service.client.search(index='opc-outages-baltics*', body=body)['hits']['hits'][0]['fields']['reportParsedDate'][0] + + # Query for latest outage UAP + uap_query = {"bool": {"must": [{"match": {"reportParsedDate": f"{last_uap_version}"}}, + {"match": {"Merge": merge_type}}]}} + uap_outages = elk_service.get_docs_by_query(index='opc-outages-baltics*', query=uap_query, size=10000) + uap_outages = uap_outages.merge(mrid_map[['eic', 'mrid']], how='left', on='eic', indicator=True).rename(columns={"mrid": 'grid_id'}) + unmapped_outages = uap_outages[uap_outages['_merge'] == 'left_only'] + + if not unmapped_outages.empty: + logger.warning(f"Unable to map following outage mRIDs: {unmapped_outages['name'].values}") + + # Filter outages according to model scenario date and replaced area + filtered_outages = uap_outages[(uap_outages['start_date'] <= scenario_datetime) & (uap_outages['end_date'] >= scenario_datetime)] + filtered_outages = filtered_outages[filtered_outages['Area'].isin(outage_areas)] + mapped_outages = filtered_outages[~filtered_outages['grid_id'].isna()] + + # Get disconnected elements in network model + model_outages = pd.DataFrame(get_model_outages(network=merged_model.network)) + mapped_model_outages = pd.merge(model_outages, mrid_map, left_on='grid_id', right_on='mrid', how='inner') + model_area_map = {"LITGRID": "LT", "AST": "LV", "ELERING": "EE"} + model_outage_areas = [model_area_map.get(item, item) for item in tso_list] + filtered_model_outages = mapped_model_outages[mapped_model_outages['country'].isin(model_outage_areas)] + + # Include cross-border lines for reconnection (both dangling lines) + dangling_lines = get_network_elements(network=merged_model.network, + element_type=pypowsybl.network.ElementType.DANGLING_LINE).reset_index(names=['grid_id']) + border_lines = dangling_lines[dangling_lines['pairing_key'].isin(model_outages['pairing_key'])] + relevant_border_lines = border_lines[border_lines['country'].isin(['LT', 'LV', 'EE'])] + additional_dangling_lines = dangling_lines[dangling_lines['pairing_key'].isin(relevant_border_lines['pairing_key'])] + + # Merged dataframe of network elements to be reconnected + filtered_model_outages = pd.concat([filtered_model_outages, additional_dangling_lines]).drop_duplicates(subset='grid_id') + filtered_model_outages = filtered_model_outages.where(pd.notnull(filtered_model_outages), None) + + # rename columns + filtered_model_outages = filtered_model_outages.copy()[['name', 'grid_id', 'eic']].rename(columns={'grid_id': 'mrid'}) + mapped_outages = mapped_outages.copy()[['name', 'grid_id', 'eic']].rename(columns={'grid_id': 'mrid'}) + + logger.info("Updating outages on merged model") + + # Reconnecting outages from network-config list + outages_updated = {} + for index, outage in filtered_model_outages.iterrows(): + try: + if merged_model.network.connect(outage['mrid']): + logger.info(f"Successfully reconnected: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages = True + outage_dict = outage.to_dict() + outage_dict.update({'status': 'connected'}) + outages_updated[outage_dict['mrid']] = outage_dict + else: + if uap_outages['grid_id'].str.contains(outage['mrid']).any(): + logger.info(f"Element is already connected: {outage['name']} [mrid: {outage['mrid']}]") + else: + logger.error(f"Failed to connect element: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + except Exception as e: + logger.error((e, outage['name'])) + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + merged_model.outages = False + continue + + # Applying outages from UAP + for index, outage in mapped_outages.iterrows(): + try: + if merged_model.network.disconnect(outage['mrid']): + logger.info(f"Successfully disconnected: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages = True + outage_dict = outage.to_dict() + outage_dict.update({'status': 'disconnected'}) + outages_updated[outage_dict['mrid']] = outage_dict + else: + if uap_outages['grid_id'].str.contains(outage['mrid']).any(): + logger.info(f"Element is already in outage: {outage['name']} [mrid: {outage['mrid']}]") + else: + logger.error(f"Failed to disconnect element: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + except Exception as e: + logger.error((e, outage['name'])) + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + merged_model.outages = False + continue + + # Keep only important keys of updated outages + merged_model.outages_updated = list(outages_updated.values()) + + if merged_model.outages_unmapped: + merged_model.outages = False + return merged_model -def remove_duplicate_sv_voltages(cgm_sv_data, original_data): - """ - Pypowsybl 1.6.0 provides multiple sets of SvVoltage values for the topological nodes that are boundary nodes (from - each IGM side that uses the corresponding boundary node). So this is a hack that removes one of them (preferably the - one that is zero). - :param cgm_sv_data: merged SV profile from where duplicate SvVoltage values are removed - :param original_data: will be used to get boundary node ids - :return updated merged SV profile - """ - # Check that models are in triplets - some_data = get_opdm_data_from_models(model_data=original_data) - # Get ids of boundary nodes that are shared by several igms - in_several_igms = (get_boundary_nodes_between_igms(model_data=some_data)) - # Get SvVoltage Ids corresponding to shared boundary nodes - sv_voltage_ids = pandas.merge(cgm_sv_data[cgm_sv_data['KEY'] == 'SvVoltage.TopologicalNode'], - in_several_igms.rename(columns={'ID': 'VALUE'}), on='VALUE') - # Get SvVoltage voltage values for corresponding SvVoltage Ids - sv_voltage_values = pandas.merge(cgm_sv_data[cgm_sv_data['KEY'] == 'SvVoltage.v'][['ID', 'VALUE']]. - rename(columns={'VALUE': 'SvVoltage.v'}), - sv_voltage_ids[['ID', 'VALUE']]. - rename(columns={'VALUE': 'SvVoltage.SvTopologicalNode'}), on='ID') - # Just in case convert the values to numeric - sv_voltage_values[['SvVoltage.v']] = (sv_voltage_values[['SvVoltage.v']].apply(lambda x: x.apply(Decimal))) - # Group by topological node id and by some logic take SvVoltage that will be dropped - voltages_to_keep = (sv_voltage_values.groupby(['SvVoltage.SvTopologicalNode']). - apply(lambda x: take_best_match_for_sv_voltage(input_data=x, - column_name='SvVoltage.v', - to_keep=True), include_groups=False)) - voltages_to_discard = sv_voltage_values.merge(voltages_to_keep['ID'], on='ID', how='left', indicator=True) - voltages_to_discard = voltages_to_discard[voltages_to_discard['_merge'] == 'left_only'] - if not voltages_to_discard.empty: - logger.info(f"Removing {len(voltages_to_discard.index)} duplicate voltage levels from boundary nodes") - sv_voltages_to_remove = pandas.merge(cgm_sv_data, voltages_to_discard['ID'].to_frame(), on='ID') - cgm_sv_data = triplets.rdf_parser.remove_triplet_from_triplet(cgm_sv_data, sv_voltages_to_remove) - return cgm_sv_data - - -def check_and_fix_dependencies(cgm_sv_data, cgm_ssh_data, original_data): - """ - Seems that pypowsybl ver 1.6.0 managed to get rid of dependencies in exported file. This gathers them from - SSH profiles and from the original models - :param cgm_sv_data: merged SV profile that is missing the dependencies - :param cgm_ssh_data: merged SSH profiles, will be used to get SSH dependencies - :param original_data: original models, will be used to get TP dependencies - :return updated merged SV profile - """ - some_data = get_opdm_data_from_models(model_data=original_data) - tp_file_ids = some_data[(some_data['KEY'] == 'Model.profile') & (some_data['VALUE'].str.contains('Topology'))] - - ssh_file_ids = cgm_ssh_data[(cgm_ssh_data['KEY'] == 'Model.profile') & - (cgm_ssh_data['VALUE'].str.contains('SteadyStateHypothesis'))] - dependencies = pandas.concat([tp_file_ids, ssh_file_ids], ignore_index=True, sort=False) - existing_dependencies = cgm_sv_data[cgm_sv_data['KEY'] == 'Model.DependentOn'] - dependency_difference = existing_dependencies.merge(dependencies[['ID']].rename(columns={'ID': 'VALUE'}), - on='VALUE', how='outer', indicator=True) - if not dependency_difference.query('_merge == "right_only"').empty: - cgm_sv_data = triplets.rdf_parser.remove_triplet_from_triplet(cgm_sv_data, existing_dependencies) - full_model_id = cgm_sv_data[(cgm_sv_data['KEY'] == 'Type') & (cgm_sv_data['VALUE'] == 'FullModel')] - dependencies_to_update = dependency_difference.query('_merge != "left_only"') - logger.info(f"Mismatch of dependencies. Inserting {len(dependencies_to_update.index)} " - f"dependencies to SV profile") - new_dependencies = dependencies_to_update[['VALUE']].copy().reset_index(drop=True) - new_dependencies.loc[:, 'KEY'] = 'Model.DependentOn' - new_dependencies.loc[:, 'ID'] = full_model_id['ID'].iloc[0] - new_dependencies.loc[:, 'INSTANCE_ID'] = full_model_id['INSTANCE_ID'].iloc[0] - cgm_sv_data = triplets.rdf_parser.update_triplet_from_triplet(cgm_sv_data, new_dependencies) - return cgm_sv_data - - -def remove_small_islands(solved_data, island_size_limit): - small_island = pandas.DataFrame(solved_data.query("KEY == 'TopologicalIsland.TopologicalNodes'").ID.value_counts()).reset_index().query("count <= @island_size_limit") - solved_data = triplets.rdf_parser.remove_triplet_from_triplet(solved_data, small_island, columns=["ID"]) - logger.info(f"Removed {len(small_island)} island(s) with size <= {island_size_limit}") - return solved_data - - -def disconnect_equipment_if_flow_sum_not_zero(cgm_sv_data, - cgm_ssh_data, - original_data, - equipment_name: str = "ConformLoad", - sv_injection_limit: float = SV_INJECTION_LIMIT): - """ - If there is a mismatch of flows at topological nodes it tries to switch of and set flows at terminals - indicated by equipment_name to original values. - The idea is that when loadflow calculation fails at some island, the results are still being updated and as - currently there is not a better way to find the islands-> nodes -> terminals on which it fails then the HACK - is to try to set them back to original values. - NOTE THAT IT NOT ONLY SETS THE VALUES TO OLD ONES BUT ALSO DISCONNECTS IT FROM TERMINAL - :param cgm_ssh_data: merged SSH profile (needed to switch the terminals of) - :param cgm_sv_data: merged SV profile (needed to set the flows for terminals) - :param original_data: IGMs (triplets, dictionary) - :param equipment_name: name of the equipment. CURRENTLY, IT IS USED FOR CONFORM LOAD - :param sv_injection_limit: threshold for deciding whether the node is violated by sum of flows - :return updated merged SV and SSH profiles - """ - original_data = get_opdm_data_from_models(model_data=original_data) - # Get power flow after lf - power_flow = cgm_sv_data.type_tableview('SvPowerFlow')[['SvPowerFlow.Terminal', 'SvPowerFlow.p', 'SvPowerFlow.q']] - # Get terminals - terminals = original_data.type_tableview('Terminal').rename_axis('Terminal').reset_index() - terminals = terminals[['Terminal', 'Terminal.ConductingEquipment', 'Terminal.TopologicalNode']] - # Calculate summed flows per topological node - flows_summed = ((power_flow.merge(terminals, left_on='SvPowerFlow.Terminal', right_on='Terminal', how='left') - .groupby('Terminal.TopologicalNode')[['SvPowerFlow.p', 'SvPowerFlow.q']] - .sum()).rename_axis('Terminal.TopologicalNode').reset_index()) - # Get topological nodes that have mismatch - nok_nodes = flows_summed[(abs(flows_summed['SvPowerFlow.p']) > sv_injection_limit) | - (abs(flows_summed['SvPowerFlow.q']) > sv_injection_limit)][['Terminal.TopologicalNode']] - # Merge terminals with summed flows at nodes - terminals_nodes = terminals.merge(flows_summed, on='Terminal.TopologicalNode', how='left') - # Get equipment names - equipment_names = (original_data.query('KEY == "Type"')[['ID', 'VALUE']] - .drop_duplicates().rename(columns={'ID': 'Terminal.ConductingEquipment', - 'VALUE': 'Equipment_name'})) - # Merge terminals with equipment names - terminals_equipment = terminals_nodes.merge(equipment_names, on='Terminal.ConductingEquipment', how='left') - # Get equipment lines corresponding to nodes that had mismatch - if not nok_nodes.empty: - logger.error(f"For {len(nok_nodes.index)} topological nodes, the sum of flows is over {sv_injection_limit}") - nok_lines = (terminals_equipment.merge(nok_nodes, on='Terminal.TopologicalNode') - .sort_values(by=['Terminal.TopologicalNode'])) - nok_loads = nok_lines[nok_lines['Equipment_name'] == equipment_name] - if not nok_loads.empty: - logger.warning(f"Switching off {len(nok_loads.index)} terminals as they contain {equipment_name}") - # Copy values from original models - old_power_flows = original_data.type_tableview('SvPowerFlow')[['SvPowerFlow.Terminal', - 'SvPowerFlow.p', 'SvPowerFlow.q']] - old_power_flows = (old_power_flows - .merge(nok_loads[['Terminal']].rename(columns={'Terminal': 'SvPowerFlow.Terminal'}), - on='SvPowerFlow.Terminal')) - new_power_flows = cgm_sv_data.type_tableview('SvPowerFlow')[['SvPowerFlow.Terminal', 'Type']] - new_power_flows = (new_power_flows.reset_index().merge(old_power_flows, on='SvPowerFlow.Terminal') - .set_index('ID')) - # Update values in SV profile - cgm_sv_data = triplets.rdf_parser.update_triplet_from_tableview(cgm_sv_data, new_power_flows) - # Just in case disconnect those things also - # The following part can cause 'Disconnected Terminal' error - # terminals_in_ssh = cgm_ssh_data[cgm_ssh_data["KEY"].str.contains("Terminal.connected")].merge( - # nok_loads[["Terminal"]].rename(columns={'Terminal': 'ID'}), on='ID') - # terminals_in_ssh.loc[:, 'VALUE'] = 'false' - # cgm_ssh_data = triplets.rdf_parser.update_triplet_from_triplet(cgm_ssh_data, terminals_in_ssh) - return cgm_sv_data, cgm_ssh_data - - -def calculate_intraday_time_horizon(scenario_datetime, task_creation_time): + +def set_intraday_time_horizon(scenario_datetime, task_creation_time): """ Finds time difference between task creation time and scenario timestamp. Converts it to hours and finds the hour number corresponding to intraday run (number of hours that scenario timestamp is ahead from task creation time) @@ -804,8 +536,8 @@ def calculate_intraday_time_horizon(scenario_datetime, task_creation_time): """ max_time_horizon_value = 36 calculated_time_horizon = '01' # DEFAULT VALUE, CHANGE THIS - _task_creation_time = helper_parse_datetime(task_creation_time, keep_timezone=False) - _scenario_datetime = helper_parse_datetime(scenario_datetime, keep_timezone=False) + _task_creation_time = parse_datetime(task_creation_time, keep_timezone=False) + _scenario_datetime = parse_datetime(scenario_datetime, keep_timezone=False) time_diff = _scenario_datetime - _task_creation_time if 0 <= time_diff.days <= 1: time_horizon_actual = math.floor((time_diff.days * 24 * 3600 + time_diff.seconds) / 3600) @@ -816,6 +548,245 @@ def calculate_intraday_time_horizon(scenario_datetime, task_creation_time): return calculated_time_horizon +def check_net_interchanges(cgm_sv_data, cgm_ssh_data, original_models, fix_errors: bool = False, threshold: float = None): + """ + An attempt to calculate the net interchange 2 values and check them against those provided in ssh profiles + :param cgm_sv_data: merged sv profile + :param cgm_ssh_data: merged ssh profile + :param original_models: igms in triplets + :param fix_errors: injects new calculated flows into merged ssh profiles + :param threshold: specify threshold if needed + :return (updated) ssh profiles + """ + try: + control_areas = (original_models.type_tableview('ControlArea') + .rename_axis('ControlArea') + .reset_index())[['ControlArea', 'ControlArea.netInterchange', 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', 'IdentifiedObject.name']] + except KeyError: + control_areas = original_models.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + ssh_areas = cgm_ssh_data.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + control_areas = control_areas.merge(ssh_areas, on='ControlArea')[['ControlArea', 'ControlArea.netInterchange', + 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', + 'IdentifiedObject.name']] + tie_flows = (original_models.type_tableview('TieFlow') + .rename_axis('TieFlow').rename(columns={'TieFlow.ControlArea': 'ControlArea', + 'TieFlow.Terminal': 'Terminal'}) + .reset_index())[['ControlArea', 'Terminal', 'TieFlow.positiveFlowIn']] + tie_flows = tie_flows.merge(control_areas[['ControlArea']], on='ControlArea') + try: + terminals = (original_models.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal', 'ACDCTerminal.connected']] + except KeyError: + terminals = (original_models.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal']] + tie_flows = tie_flows.merge(terminals, on='Terminal') + try: + power_flows_pre = (original_models.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + tie_flows = tie_flows.merge(power_flows_pre, on='Terminal', how='left') + except Exception as error: + logger.error(f"Was not able to get tie flows from original models with exception: {error}") + power_flows_post = (cgm_sv_data.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + + tie_flows = tie_flows.merge(power_flows_post, on='Terminal', how='left', + suffixes=('_pre', '_post')) + try: + tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p_pre', 'SvPowerFlow.p_post']] + .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) + .rename_axis('ControlArea').reset_index()) + except KeyError: + tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p']] + .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) + .rename_axis('ControlArea').reset_index()) + tie_flows_grouped = tie_flows_grouped.rename(columns={'SvPowerFlow.p': 'SvPowerFlow.p_post'}) + tie_flows_grouped = control_areas.merge(tie_flows_grouped, on='ControlArea') + if threshold and threshold > 0: + tie_flows_grouped['Exceeded'] = (abs(tie_flows_grouped['ControlArea.netInterchange'] + - tie_flows_grouped['SvPowerFlow.p_post']) > threshold) + else: + tie_flows_grouped['Exceeded'] = (abs(tie_flows_grouped['ControlArea.netInterchange'] + - tie_flows_grouped['SvPowerFlow.p_post']) > + tie_flows_grouped['ControlArea.pTolerance']) + net_interchange_errors = tie_flows_grouped[tie_flows_grouped.eval('Exceeded')] + if not net_interchange_errors.empty: + logger.warning(f"Found {len(net_interchange_errors.index)} possible net interchange_2 problems over {threshold}") + # Apply modification + if fix_errors: + logger.warning(f"Updating {len(net_interchange_errors.index)} interchanges to new values") + new_areas = cgm_ssh_data.type_tableview('ControlArea').reset_index()[['ID', + 'ControlArea.pTolerance', 'Type']] + new_areas = new_areas.merge(net_interchange_errors[['ControlArea', 'SvPowerFlow.p_post']] + .rename(columns={'ControlArea': 'ID', + 'SvPowerFlow.p_post': 'ControlArea.netInterchange'}), on='ID') + cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(cgm_ssh_data, new_areas) + + return cgm_ssh_data + + +def check_non_boundary_equivalent_injections(cgm_sv_data, + cgm_ssh_data, + original_models, + threshold: float = 0, + fix_errors: bool = False): + """ + Checks equivalent injections that are not on boundary topological nodes + :param cgm_sv_data: merged SV profile + :param cgm_ssh_data: merged SSH profile + :param original_models: igms in triplets + :param threshold: threshold for checking + :param fix_errors: if true then copies values from sv profile to ssh profile + :return cgm_ssh_data + """ + boundary_nodes = original_models.query('KEY == "TopologicalNode.boundaryPoint" & VALUE == "true"')[['ID']] + terminals = (original_models.type_tableview('Terminal').rename_axis('SvPowerFlow.Terminal').reset_index() + .merge(boundary_nodes.rename(columns={'ID': 'Terminal.TopologicalNode'}), + on='Terminal.TopologicalNode', how='outer', indicator=True))[['SvPowerFlow.Terminal', + 'Terminal.ConductingEquipment', + '_merge']] + terminals = terminals[terminals['_merge'] == 'left_only'][['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] + return check_all_kind_of_injections(cgm_sv_data=cgm_sv_data, + cgm_ssh_data=cgm_ssh_data, + original_models=original_models, + injection_name='EquivalentInjection', + fields_to_check={'SvPowerFlow.p': 'EquivalentInjection.p'}, + threshold=threshold, + terminals=terminals, + fix_errors=fix_errors) + + +def check_all_kind_of_injections(cgm_sv_data, + cgm_ssh_data, + original_models, + injection_name: str = 'ExternalNetworkInjection', + fields_to_check: dict = None, + fix_errors: bool = False, + threshold: float = 0, + terminals: pd.DataFrame = None, + report_sum: bool = True): + """ + Compares the given cgm ssh injection values to the corresponding sv powerflow values in cgm sv + :param cgm_sv_data: merged SV profile + :param cgm_ssh_data: merged SSH profile + :param original_models: igms in triplets + :param injection_name: name of the injection + :param fields_to_check: dictionary where key is the field in powerflow and value is the field in injection + :param fix_errors: if true then copies values from sv profile to ssh profile + :param threshold: max allowed mismatch + :param terminals: optional, can give dataframe of terminals as input + :param report_sum: if true prints sum of injections and powerflows to console + :return cgm_ssh_data + """ + if not fields_to_check: + return cgm_ssh_data + + fixed_fields = ['ID'] + try: + original_injections = original_models.type_tableview(injection_name).reset_index() + injections = cgm_ssh_data.type_tableview(injection_name).reset_index() + except AttributeError: + logger.info(f"SSH profile doesn't contain data about {injection_name}") + return cgm_ssh_data + injections_reduced = injections[[*fixed_fields, *fields_to_check.values()]] + original_injections_reduced = original_injections[[*fixed_fields, *fields_to_check.values()]] + injections_reduced = injections_reduced.merge(original_injections_reduced, on='ID', suffixes=('', '_org')) + if terminals is None: + terminals = (original_models.type_tableview('Terminal') + .rename_axis('SvPowerFlow.Terminal') + .reset_index())[['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] + flows = (cgm_sv_data.type_tableview('SvPowerFlow') + .reset_index())[[*['SvPowerFlow.Terminal'], *fields_to_check.keys()]] + terminals = terminals.merge(flows, on='SvPowerFlow.Terminal') + terminals = terminals.merge(injections_reduced, left_on='Terminal.ConductingEquipment', right_on='ID') + + filtered_list = [] + for flow_field, injection_field in fields_to_check.items(): + filtered_list.append(terminals[abs(terminals[injection_field] - terminals[flow_field]) > threshold]) + if report_sum: + logger.info(f"IGM {injection_field} = {terminals[injection_field + '_org'].sum()} vs " + f"CGM {injection_field} = {terminals[injection_field].sum()} vs " + f"CGM {flow_field} = {terminals[flow_field].sum()}") + if not filtered_list: + return cgm_ssh_data + + filtered = pd.concat(filtered_list).drop_duplicates().reset_index(drop=True) + if not filtered.empty: + logger.warning(f"Found {len(filtered.index)} mismatches between {injection_name} and flow values on terminals") + # Apply modification + if fix_errors: + logger.info(f"Updating {injection_name} values from terminal flow values") + injections_update = injections.merge(filtered[[*fixed_fields, *fields_to_check.keys()]]) + injections_update = injections_update.drop(columns=fields_to_check.values()) + injections_update = injections_update.rename(columns=fields_to_check) + cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(data=cgm_ssh_data, + tableview=injections_update, + update=True, + add=False) + return cgm_ssh_data + + +def run_post_merge_processing(input_models: list, + exported_model: bytes, + opdm_object_meta: dict, + enable_temp_fixes: bool, + ): + + # Load original input models to triplets + input_models_triplets = load_opdm_objects_to_triplets(opdm_objects=input_models) + + # Apply corrections to SV profile + update_merged_model_sv(sv_data=exported_model, opdm_object_meta=opdm_object_meta) + + # Create update SSH + sv_data, ssh_data, opdm_object_meta = create_updated_ssh() + fix_net_interchange_errors = task_properties.get('fix_net_interchange2', False) + + # Run temporary modification on exported model + if enable_temp_fixes: + # TODO need to revise constantly + sv_data = temporary.remove_equivalent_shunt_section(sv_data, input_models_triplets) + sv_data = temporary.add_missing_sv_tap_steps(sv_data, ssh_data) + sv_data = temporary.remove_small_islands(sv_data, int(SMALL_ISLAND_SIZE)) + sv_data = temporary.remove_duplicate_sv_voltages(cgm_sv_data=sv_data, original_data=input_models_triplets) + sv_data = temporary.check_and_fix_dependencies(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=input_models_triplets) + + # Run injections check and apply modification if defined in configuration + ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, + cgm_sv_data=sv_data, + original_models=input_models_triplets, + injection_name='EnergySource', + threshold=INJECTION_THRESHOLD, + fields_to_check={'SvPowerFlow.p': 'EnergySource.activePower'}, + fix_errors=FIX_INJECTION_ERRORS) + ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, + cgm_sv_data=sv_data, + original_models=input_models_triplets, + injection_name='ExternalNetworkInjection', + fields_to_check={'SvPowerFlow.p': 'ExternalNetworkInjection.p'}, + threshold=INJECTION_THRESHOLD, + fix_errors=FIX_INJECTION_ERRORS) + ssh_data = check_non_boundary_equivalent_injections(cgm_sv_data=sv_data, + cgm_ssh_data=ssh_data, + original_models=input_models_triplets, + threshold=INJECTION_THRESHOLD, + fix_errors=FIX_INJECTION_ERRORS) + + try: + ssh_data = check_net_interchanges(cgm_sv_data=sv_data, + cgm_ssh_data=ssh_data, + original_models=input_models_triplets, + fix_errors=fix_net_interchange_errors, + threshold=NET_INTERCHANGE_THRESHOLD) + except KeyError: + logger.warning(f"No fields for net interchange correction") + + return sv_data, ssh_data, opdm_object_meta + + if __name__ == "__main__": from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index f4d46943..2fcf719c 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -1,5 +1,4 @@ import logging -import pandas import pypowsybl import config import json @@ -8,25 +7,27 @@ import datetime from dataclasses import dataclass, field from typing import List - -from emf.common.time_helper import parse_datetime +from emf.common.helpers.time import parse_datetime from io import BytesIO from zipfile import ZipFile from emf.common.config_parser import parse_app_properties -from emf.common.integrations import opdm, minio_api, elastic, rabbit +from emf.common.integrations import opdm, minio_api, elastic from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download from emf.common.integrations.object_storage.schedules import query_acnp_schedules, query_hvdc_schedules from emf.common.loadflow_tool import loadflow_settings -from emf.common.loadflow_tool.helper import opdmprofile_to_bytes, attr_to_dict, export_to_cgmes_zip +from emf.common.helpers.utils import attr_to_dict +from emf.common.helpers.cgmes import export_to_cgmes_zip +from emf.common.helpers.opdm_objects import get_opdm_component_data_bytes +from emf.common.helpers.loadflow import load_network_model +from emf.common.helpers.tasks import update_task_status from emf.model_merger import merge_functions from emf.model_merger import scaler from emf.model_merger.replacement import run_replacement, get_tsos_available_in_storage -from emf.task_generator.task_generator import update_task_status +from emf.model_merger.temporary import handle_igm_ssh_vs_cgm_ssh_error from emf.common.logging.custom_logger import get_elk_logging_handler -# TODO - move this async solution to some common module from concurrent.futures import ThreadPoolExecutor from lxml import etree -from emf.model_merger.temporary_fixes import run_post_merge_processing, fix_model_outages, fix_igm_ssh_vs_cgm_ssh_error + logger = logging.getLogger(__name__) parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.merger) @@ -90,6 +91,34 @@ def __init__(self): self.minio_service = minio_api.ObjectStorage() self.elk_logging_handler = get_elk_logging_handler() + @staticmethod + def run_loadflow(merged_model): + # report = pypowsybl.report.Reporter() + result = pypowsybl.loadflow.run_ac(network=merged_model.network, + parameters=getattr(loadflow_settings, MERGE_LOAD_FLOW_SETTINGS), + # reporter=loadflow_report, + ) + + result_dict = [attr_to_dict(island) for island in result] + # Modify all nested objects to native data types + for island in result_dict: + island['status'] = island['status'].name + # Extract only first slack bus from internal pypowsybl object + slack_bus_results = island.pop('slack_bus_results') + if slack_bus_results: + island['slack_bus_id'] = getattr(slack_bus_results[0], 'id', 'undefined') + island['active_power_mismatch'] = getattr(slack_bus_results[0], 'active_power_mismatch', float()) + else: + island['slack_bus_id'] = 'undefined' + island['active_power_mismatch'] = float() + + # merged_model.loadflow = json.loads(loadflow_report.to_json()) + # merged_model.loadflow = str(loadflow_report) + merged_model.loadflow = [island for island in result_dict if island['reference_bus_id']] + merged_model.loadflow_status = result[0].status.name # store main island loadflow status + + return merged_model + def handle(self, task_object: dict, properties: dict, **kwargs): start_time = datetime.datetime.now(datetime.UTC) @@ -134,20 +163,8 @@ def handle(self, task_object: dict, properties: dict, **kwargs): model_upload_to_opdm = task_properties["upload_to_opdm"] model_upload_to_minio = task_properties["upload_to_minio"] model_merge_report_send_to_elk = task_properties["send_merge_report"] - pre_temp_fixes = task_properties['pre_temp_fixes'] post_temp_fixes = task_properties['post_temp_fixes'] force_outage_fix = task_properties['force_outage_fix'] - open_non_retained_switches_between_tn = json.loads(OPEN_NON_RETAINED_SWITCHES_BETWEEN_TN.lower()) - - # TODO used for temp fixes - consider move it out somewhere - try: - net_interchange_threshold = int(NET_INTERCHANGE_THRESHOLD) - except Exception: - net_interchange_threshold = 200 - - task_properties['net_interchange2_threshold'] = net_interchange_threshold - - remove_non_generators_from_slack_participation = True # TODO move to conf # Collect valid models from ObjectStorage downloaded_models = get_latest_models_and_download(time_horizon, scenario_datetime, valid=False, data_source='OPDM') @@ -247,7 +264,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): # Load network model and merge merge_start = datetime.datetime.now(datetime.UTC) - merged_model.network = merge_functions.load_model(opdm_objects=input_models) + merged_model.network = load_network_model(opdm_objects=input_models) merged_model.network_meta = attr_to_dict(instance=merged_model.network, sanitize_to_strings=True) merged_model.included = [model['pmd:TSO'] for model in valid_models if model['pmd:TSO']] @@ -262,23 +279,22 @@ def handle(self, task_object: dict, properties: dict, **kwargs): elif merging_area == 'BA' and any(tso in ['LITGRID', 'AST', 'ELERING'] for tso in replaced_tso_list): # by default do it on Baltic merge replaced models tso_list = replaced_tso_list if tso_list: # if not set force and not replaced BA then nothing to fix - merged_model = fix_model_outages(merged_model=merged_model, - tso_list=tso_list, - scenario_datetime=scenario_datetime, - time_horizon=time_horizon) - - # Various fixes from igmsshvscgmssh error - if remove_non_generators_from_slack_participation: - merged_model.network = fix_igm_ssh_vs_cgm_ssh_error(merged_model.network) + merged_model = merge_functions.update_model_outages(merged_model=merged_model, + tso_list=tso_list, + scenario_datetime=scenario_datetime, + time_horizon=time_horizon) - # TODO - placeholder for open TN switcher + # Various corrections from igmsshvscgmssh error + if json.loads(REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION.lower()): + merged_model.network = handle_igm_ssh_vs_cgm_ssh_error(network_pre_instance=merged_model.network) - # TODO placeholder for configure paired tie lines + # TODO - placeholder for open TN switcher or it can be after scaling before export + if json.loads(OPEN_NON_RETAINED_SWITCHES_BETWEEN_TN.lower()): + pass # TODO - run other LF if default fails # Run loadflow on merged model - merged_model = merge_functions.run_lf(merged_model=merged_model, - loadflow_settings=getattr(loadflow_settings, MERGE_LOAD_FLOW_SETTINGS)) + merged_model = self.run_loadflow(merged_model=merged_model) logger.info(f"Loadflow status of main island: {merged_model.loadflow[0]['status_text']}") # Perform scaling @@ -306,29 +322,52 @@ def handle(self, task_object: dict, properties: dict, **kwargs): logger.error(e) merged_model.scaled = False else: - logger.warning(f"Schedule reference data not available: {schedule_time_horizon} for {schedule_start}, skipping model scaling") + logger.warning(f"Schedule reference data not available: {schedule_time_horizon} for {schedule_start}") + logger.warning(f"Model schedule scaling not performed") merged_model.scaled = False # Record main merging process end merge_end = datetime.datetime.now(datetime.UTC) # Update time_horizon in case of generic ID process type - new_time_horizon = None if time_horizon.upper() == "ID": - time_horizon = merge_functions.calculate_intraday_time_horizon(scenario_datetime, task_creation_time) - new_time_horizon = time_horizon + time_horizon = merge_functions.set_intraday_time_horizon(scenario_datetime, task_creation_time) logger.info(f"Setting intraday time horizon to: {time_horizon}") + # Set merged model name + model_type = "RMM" if merging_area == "BA" else "CGM" + merged_model.name = f"{model_type}_{time_horizon}_{version}_{parse_datetime(scenario_datetime):%Y%m%dT%H%MZ}_{merging_area}_{uuid4()}" + + # Crate OPDM object for merged model + opdm_object_meta = merge_functions.create_merged_model_opdm_object( + object_id=merged_model.network_meta['id'].split("uuid:")[-1], + time_horizon=time_horizon, + merging_entity=merging_entity, + merging_area=merging_area, + scenario_date=scenario_datetime, + mas=mas, + version=version, + ) + + # Ensure boundary point EquivalentInjection are set to zero for paired tie lines + merged_model.network = merge_functions.ensure_paired_equivalent_injection_compatibility(network=merged_model.network) + + # Export merged model + # TODO change here to export SSH profiles as well + exported_model = merge_functions.export_merged_model(network=merged_model.network, + opdm_object_meta=opdm_object_meta, + profiles=["SV"], + cgm_convention=False) + # Run post-processing post_p_start = datetime.datetime.now(datetime.UTC) logger.info(f"Starting merged model post-processing") # TODO here should be one existing network structure. IIDM model can be exported and removed to release memory - sv_data, ssh_data, opdm_object_meta = run_post_merge_processing(input_models=input_models, - merged_model=merged_model, - task_properties=task_properties, - small_island_size=SMALL_ISLAND_SIZE, - enable_temp_fixes=post_temp_fixes, - time_horizon=new_time_horizon) + sv_data, ssh_data, opdm_object_meta = merge_functions.run_post_merge_processing(input_models=input_models, + exported_model=exported_model, + opdm_object_meta=opdm_object_meta, + enable_temp_fixes=post_temp_fixes, + ) # Package both input models and exported CGM profiles to in memory zip files serialized_data = export_to_cgmes_zip([ssh_data, sv_data]) @@ -350,12 +389,8 @@ def handle(self, task_object: dict, properties: dict, **kwargs): logging.error(f"Unexpected error on uploading to OPDM: {error}", exc_info=True) # Create zipped model data - if merging_area == 'BA': - merged_model.name = f"RMM_{time_horizon}_{version}_{parse_datetime(scenario_datetime):%Y%m%dT%H%MZ}_{merging_area}_{uuid4()}" - else: - merged_model.name = f"CGM_{time_horizon}_{version}_{parse_datetime(scenario_datetime):%Y%m%dT%H%MZ}_{merging_area}_{uuid4()}" - merged_model_data = BytesIO() - with ZipFile(merged_model_data, "w") as merged_model_zip: + merged_model_object = BytesIO() + with ZipFile(merged_model_object, "w") as merged_model_zip: # Include CGM model files for item in serialized_data: merged_model_zip.writestr(item.name, item.getvalue()) @@ -364,11 +399,9 @@ def handle(self, task_object: dict, properties: dict, **kwargs): for input_model in input_models: for instance in input_model['opde:Component']: if instance['opdm:Profile']['pmd:cgmesProfile'] in ['EQ', 'TP', 'EQBD', 'TPBD']: - file_object = opdmprofile_to_bytes(instance) + file_object = get_opdm_component_data_bytes(opdm_component=instance) logging.info(f"Adding file: {file_object.name}") merged_model_zip.writestr(file_object.name, file_object.getvalue()) - - merged_model_object = merged_model_data merged_model_object.name = f"{OUTPUT_MINIO_FOLDER}/{merged_model.name}.zip" # Upload to Minio storage @@ -404,7 +437,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): merged_model.content_reference = merged_model_object.name # Update OPDM object data with CGM relevant data and send to Elastic - opdm_object_meta = merge_functions.update_cgm_opdm_object_meta(opdm_object_meta, merged_model) + opdm_object_meta['pmd:content-reference'] = merged_model.content_reference response = elastic.Elastic.send_to_elastic(index=OPDE_MODELS_ELK_INDEX, json_message=opdm_object_meta) # Send merge report and opdm object metadata to Elastic @@ -457,10 +490,10 @@ def handle(self, task_object: dict, properties: dict, **kwargs): "job_period_start": "2024-05-24T22:00:00+00:00", "job_period_end": "2024-05-25T06:00:00+00:00", "task_properties": { - "timestamp_utc": "2025-06-12T12:30:00+00:00", - "merge_type": "EU", + "timestamp_utc": "2025-06-16T12:30:00+00:00", + "merge_type": "BA", "merging_entity": "BALTICRCC", - "included": [], + "included": ["LITGRID", "AST", "ELERING", "PSE"], "excluded": [], "local_import": [], "time_horizon": "1D", diff --git a/emf/model_merger/scaler.py b/emf/model_merger/scaler.py index cc7c44ab..dd4f8241 100644 --- a/emf/model_merger/scaler.py +++ b/emf/model_merger/scaler.py @@ -31,8 +31,8 @@ from emf.common.config_parser import parse_app_properties from emf.common.decorators import performance_counter from emf.common.integrations.object_storage.schedules import query_acnp_schedules, query_hvdc_schedules -from emf.common.loadflow_tool.helper import attr_to_dict, get_network_elements, get_slack_generators, \ - get_connected_components_data +from emf.common.helpers.utils import attr_to_dict +from emf.common.helpers.loadflow import get_network_elements, get_slack_generators, get_connected_components_data from emf.common.loadflow_tool.loadflow_settings import CGM_RELAXED_1 logger = logging.getLogger(__name__) diff --git a/emf/model_merger/temporary.py b/emf/model_merger/temporary.py new file mode 100644 index 00000000..aa800970 --- /dev/null +++ b/emf/model_merger/temporary.py @@ -0,0 +1,400 @@ +import pypowsybl +import triplets +import pandas as pd +import logging +import uuid +from decimal import Decimal +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets + +logger = logging.getLogger(__name__) + + +def remove_small_islands(solved_data, island_size_limit): + # TODO - EVALUATE LEGACY + small_island = pd.DataFrame(solved_data.query("KEY == 'TopologicalIsland.TopologicalNodes'").ID.value_counts()).reset_index().query("count <= @island_size_limit") + solved_data = triplets.rdf_parser.remove_triplet_from_triplet(solved_data, small_island, columns=["ID"]) + logger.info(f"Removed {len(small_island)} island(s) with size <= {island_size_limit}") + return solved_data + + +def remove_equivalent_shunt_section(sv_data: pd.DataFrame, models_as_triplets: pd.DataFrame): + # TODO - EVALUATE LEGACY + """Remove Shunt Sections for EQV Shunts from SV profile""" + + equiv_shunt = models_as_triplets.query("KEY == 'Type' and VALUE == 'EquivalentShunt'") + if len(equiv_shunt) > 0: + shunts_to_remove = sv_data.merge( + sv_data.query("KEY == 'SvShuntCompensatorSections.ShuntCompensator'").merge(equiv_shunt.ID, left_on='VALUE', + right_on="ID", how='inner', + suffixes=('', '_EQVShunt')).ID) + if len(shunts_to_remove) > 0: + logger.warning(f'Removing invalid SvShuntCompensatorSections for EquivalentShunt') + sv_data = triplets.rdf_parser.remove_triplet_from_triplet(sv_data, shunts_to_remove) + + return sv_data + + +def add_missing_sv_tap_steps(sv_data: pd.DataFrame, ssh_data: pd.DataFrame): + # TODO - EVALUATE LEGACY + """Update missing tap changer tap steps in SV""" + + ssh_tap_steps = ssh_data.query("KEY == 'TapChanger.step'") + sv_tap_steps = sv_data.query("KEY == 'SvTapStep.TapChanger'") + + missing_sv_tap_steps = ssh_tap_steps.merge(sv_tap_steps[['VALUE']], + left_on='ID', + right_on="VALUE", + how='left', + indicator=True, + suffixes=('', '_SV')).query("_merge == 'left_only'") + + tap_steps_to_be_added = [] + SV_INSTANCE_ID = sv_data.INSTANCE_ID.iloc[0] + for tap_changer in missing_sv_tap_steps.itertuples(): + ID = str(uuid.uuid4()) + logger.warning(f'Missing SvTapStep for {tap_changer.ID}, adding SvTapStep {ID} and taking tap value {tap_changer.VALUE} from SSH') + tap_steps_to_be_added.extend([ + (ID, 'Type', 'SvTapStep', SV_INSTANCE_ID), + (ID, 'SvTapStep.TapChanger', tap_changer.ID, SV_INSTANCE_ID), + (ID, 'SvTapStep.position', tap_changer.VALUE, SV_INSTANCE_ID), + ]) + + sv_data = pd.concat([sv_data, pd.DataFrame(tap_steps_to_be_added, columns=['ID', 'KEY', 'VALUE', 'INSTANCE_ID'])], ignore_index=True) + + return sv_data + + +def open_switches_in_network(network_pre_instance: pypowsybl.network.Network, switches_dataframe: pd.DataFrame): + # TODO - EVALUATE LEGACY + """ + Opens switches in loaded network given by dataframe (uses ID for merging) + :param network_pre_instance: pypowsybl Network instance where igms are loaded in + :param switches_dataframe: dataframe + """ + logger.info(f"Opening {len(switches_dataframe.index)} switches") + switches = network_pre_instance.get_switches(all_attributes=True).reset_index() + switches = switches.merge(switches_dataframe[['ID']].rename(columns={'ID': 'id'}), on='id') + non_retained_closed = switches.merge(switches_dataframe.rename(columns={'ID': 'id'}), + on='id')[['id', 'open']] + non_retained_closed['open'] = True + network_pre_instance.update_switches(non_retained_closed.set_index('id')) + return network_pre_instance + + +def check_and_fix_dependencies(cgm_sv_data, cgm_ssh_data, original_data): + # TODO - EVALUATE LEGACY + """ + Seems that pypowsybl ver 1.6.0 managed to get rid of dependencies in exported file. This gathers them from + SSH profiles and from the original models + :param cgm_sv_data: merged SV profile that is missing the dependencies + :param cgm_ssh_data: merged SSH profiles, will be used to get SSH dependencies + :param original_data: original models, will be used to get TP dependencies + :return updated merged SV profile + """ + some_data = load_opdm_objects_to_triplets(opdm_objects=original_data) + tp_file_ids = some_data[(some_data['KEY'] == 'Model.profile') & (some_data['VALUE'].str.contains('Topology'))] + + ssh_file_ids = cgm_ssh_data[(cgm_ssh_data['KEY'] == 'Model.profile') & + (cgm_ssh_data['VALUE'].str.contains('SteadyStateHypothesis'))] + dependencies = pd.concat([tp_file_ids, ssh_file_ids], ignore_index=True, sort=False) + existing_dependencies = cgm_sv_data[cgm_sv_data['KEY'] == 'Model.DependentOn'] + dependency_difference = existing_dependencies.merge(dependencies[['ID']].rename(columns={'ID': 'VALUE'}), + on='VALUE', how='outer', indicator=True) + if not dependency_difference.query('_merge == "right_only"').empty: + cgm_sv_data = triplets.rdf_parser.remove_triplet_from_triplet(cgm_sv_data, existing_dependencies) + full_model_id = cgm_sv_data[(cgm_sv_data['KEY'] == 'Type') & (cgm_sv_data['VALUE'] == 'FullModel')] + dependencies_to_update = dependency_difference.query('_merge != "left_only"') + logger.info(f"Mismatch of dependencies. Inserting {len(dependencies_to_update.index)} " + f"dependencies to SV profile") + new_dependencies = dependencies_to_update[['VALUE']].copy().reset_index(drop=True) + new_dependencies.loc[:, 'KEY'] = 'Model.DependentOn' + new_dependencies.loc[:, 'ID'] = full_model_id['ID'].iloc[0] + new_dependencies.loc[:, 'INSTANCE_ID'] = full_model_id['INSTANCE_ID'].iloc[0] + cgm_sv_data = triplets.rdf_parser.update_triplet_from_triplet(cgm_sv_data, new_dependencies) + return cgm_sv_data + + +def handle_igm_ssh_vs_cgm_ssh_error(network_pre_instance: pypowsybl.network.Network): + # TODO - EVALUATE LEGACY + """ + Implements various fixes to suppress igm ssh vs cgm ssh error + 1) Get all generators and remove them from slack distribution + 2) If generators have target_p outside the endpoints ('limits') of a curve then set it to be within + 3) Condensers p should not be modified so if it is not 0 then it sets the target_p to equal the existing p + :param network_pre_instance: pypowsybl Network instance where igms are loaded in + :return updated network_pre_instance + """ + try: + all_generators = network_pre_instance.get_elements(element_type=pypowsybl.network.ElementType.GENERATOR, + all_attributes=True).reset_index() + generators_mask = (all_generators['CGMES.synchronousMachineOperatingMode'].str.contains('generator')) + not_generators = all_generators[~generators_mask] + generators = all_generators[generators_mask] + curve_points = (network_pre_instance + .get_elements(element_type=pypowsybl.network.ElementType.REACTIVE_CAPABILITY_CURVE_POINT, + all_attributes=True).reset_index()) + curve_limits = (curve_points.merge(generators[['id']], on='id') + .groupby('id').agg(curve_p_min=('p', 'min'), curve_p_max=('p', 'max'))).reset_index() + curve_generators = generators.merge(curve_limits, on='id') + # low end can be zero + curve_generators = curve_generators[(curve_generators['target_p'] > curve_generators['curve_p_max']) | + ((curve_generators['target_p'] > 0) & + (curve_generators['target_p'] < curve_generators['curve_p_min']))] + if not curve_generators.empty: + logger.warning(f"Found {len(curve_generators.index)} generators for " + f"which p > max(reactive capacity curve(p)) or p < min(reactive capacity curve(p))") + + # Solution 1: set max_p from curve max, it should contain p on target-p + upper_limit_violated = curve_generators[(curve_generators['max_p'] > curve_generators['curve_p_max'])] + if not upper_limit_violated.empty: + logger.warning(f"Updating max p from curve for {len(upper_limit_violated.index)} generators") + upper_limit_violated['max_p'] = upper_limit_violated['curve_p_max'] + network_pre_instance.update_generators(upper_limit_violated[['id', 'max_p']].set_index('id')) + + lower_limit_violated = curve_generators[(curve_generators['min_p'] < curve_generators['curve_p_min'])] + if not lower_limit_violated.empty: + logger.warning(f"Updating min p from curve for {len(lower_limit_violated.index)} generators") + lower_limit_violated['min_p'] = lower_limit_violated['curve_p_min'] + network_pre_instance.update_generators(lower_limit_violated[['id', 'min_p']].set_index('id')) + + # Solution 2: discard generator from participating + extensions = network_pre_instance.get_extensions('activePowerControl') + remove_curve_generators = extensions.merge(curve_generators[['id']], + left_index=True, right_on='id') + if not remove_curve_generators.empty: + remove_curve_generators['participate'] = False + network_pre_instance.update_extensions('activePowerControl', + remove_curve_generators.set_index('id')) + condensers = all_generators[(all_generators['CGMES.synchronousMachineType'].str.contains('condenser')) + & (abs(all_generators['p']) > 0) + & (abs(all_generators['target_p']) == 0)] + # Fix condensers that have p not zero by setting their target_p to equal to p + if not condensers.empty: + logger.warning(f"Found {len(condensers.index)} condensers for which p ~= 0 & target_p = 0") + condensers.loc[:, 'target_p'] = condensers['p'] * (-1) + network_pre_instance.update_generators(condensers[['id', 'target_p']].set_index('id')) + # Remove all not generators from active power distribution + if not not_generators.empty: + logger.warning(f"Removing {len(not_generators.index)} machines from power distribution") + extensions = network_pre_instance.get_extensions('activePowerControl') + remove_not_generators = extensions.merge(not_generators[['id']], left_index=True, right_on='id') + remove_not_generators['participate'] = False + remove_not_generators = remove_not_generators.set_index('id') + network_pre_instance.update_extensions('activePowerControl', remove_not_generators) + except Exception as ex: + logger.warning(f"Unable to pre-process for igm-cgm-ssh error: {ex}") + + return network_pre_instance + + +def take_best_match_for_sv_voltage(input_data, column_name: str = 'v', to_keep: bool = True): + # TODO - EVALUATE LEGACY + """ + Returns one row for with sv voltage id for topological node + 1) Take the first + 2) If first is zero take first non-zero row if exists + :param input_data: input dataframe + :param column_name: name of the column + :param to_keep: either to keep or discard a value + """ + first_row = input_data.iloc[0] + if to_keep: + remaining_rows = input_data[input_data[column_name] != 0] + if first_row[column_name] == 0 and not remaining_rows.empty: + first_row = remaining_rows.iloc[0] + else: + remaining_rows = input_data[input_data[column_name] == 0] + if first_row[column_name] != 0 and not remaining_rows.empty: + first_row = remaining_rows.iloc[0] + return first_row + + +def get_boundary_nodes_between_igms(model_data: list | pd.DataFrame): + # TODO - EVALUATE LEGACY + """ + Filters out nodes that are between the igms (mentioned at least 2 igms) + :param model_data: input models + : return series of node ids + """ + model_data = get_opdm_data_from_models(model_data=model_data) + all_boundary_nodes = model_data[(model_data['KEY'] == 'TopologicalNode.boundaryPoint') & + (model_data['VALUE'] == 'true')] + # Get boundary nodes that exist in igms + merged = pd.merge(all_boundary_nodes, + model_data[(model_data['KEY'] == 'SvVoltage.TopologicalNode')], + left_on='ID', right_on='VALUE', suffixes=('_y', '')) + # Get duplicates (all of them) then duplicated values. keep=False marks all duplicates True, 'first' marks first + # occurrence to false, 'last' marks last occurrence to false. If any of them is used then in case duplicates are 2 + # then 1 is retrieved, if duplicates >3 then duplicates-1 retrieved. So, get all the duplicates and as a second + # step, drop the duplicates + merged = (merged[merged.duplicated(['VALUE'], keep=False)]).drop_duplicates(subset=['VALUE']) + in_several_igms = (merged["VALUE"]).to_frame().rename(columns={'VALUE': 'ID'}) + return in_several_igms + + +def remove_duplicate_sv_voltages(cgm_sv_data, original_data): + # TODO - EVALUATE LEGACY + """ + Pypowsybl 1.6.0 provides multiple sets of SvVoltage values for the topological nodes that are boundary nodes (from + each IGM side that uses the corresponding boundary node). So this is a hack that removes one of them (preferably the + one that is zero). + :param cgm_sv_data: merged SV profile from where duplicate SvVoltage values are removed + :param original_data: will be used to get boundary node ids + :return updated merged SV profile + """ + # Check that models are in triplets + some_data = get_opdm_data_from_models(model_data=original_data) + # Get ids of boundary nodes that are shared by several igms + in_several_igms = (get_boundary_nodes_between_igms(model_data=some_data)) + # Get SvVoltage Ids corresponding to shared boundary nodes + sv_voltage_ids = pd.merge(cgm_sv_data[cgm_sv_data['KEY'] == 'SvVoltage.TopologicalNode'], + in_several_igms.rename(columns={'ID': 'VALUE'}), on='VALUE') + # Get SvVoltage voltage values for corresponding SvVoltage Ids + sv_voltage_values = pd.merge(cgm_sv_data[cgm_sv_data['KEY'] == 'SvVoltage.v'][['ID', 'VALUE']]. + rename(columns={'VALUE': 'SvVoltage.v'}), + sv_voltage_ids[['ID', 'VALUE']]. + rename(columns={'VALUE': 'SvVoltage.SvTopologicalNode'}), on='ID') + # Just in case convert the values to numeric + sv_voltage_values[['SvVoltage.v']] = (sv_voltage_values[['SvVoltage.v']].apply(lambda x: x.apply(Decimal))) + # Group by topological node id and by some logic take SvVoltage that will be dropped + voltages_to_keep = (sv_voltage_values.groupby(['SvVoltage.SvTopologicalNode']). + apply(lambda x: take_best_match_for_sv_voltage(input_data=x, + column_name='SvVoltage.v', + to_keep=True), include_groups=False)) + voltages_to_discard = sv_voltage_values.merge(voltages_to_keep['ID'], on='ID', how='left', indicator=True) + voltages_to_discard = voltages_to_discard[voltages_to_discard['_merge'] == 'left_only'] + if not voltages_to_discard.empty: + logger.info(f"Removing {len(voltages_to_discard.index)} duplicate voltage levels from boundary nodes") + sv_voltages_to_remove = pd.merge(cgm_sv_data, voltages_to_discard['ID'].to_frame(), on='ID') + cgm_sv_data = triplets.rdf_parser.remove_triplet_from_triplet(cgm_sv_data, sv_voltages_to_remove) + + return cgm_sv_data + + +################################################### LEGACY FUNCTIONS ################################################### +def configure_paired_boundarypoint_injections(data): + # TODO [LEGACY] + """Where there are paired boundary points, equivalent injections need to be modified + Set P and Q to 0 - so that no additional consumption or production is on tie line + Set voltage control off - so that no additional consumption or production is on tie line + Set terminal to connected - to be sure we have paired connected injections at boundary point + """ + boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] + boundary_points = boundary_points.merge(data.type_tableview("Terminal").reset_index(), + left_on="ID", + right_on="Terminal.ConnectivityNode", + suffixes=('_ConnectivityNode', '_Terminal')) + injections = data.type_tableview('EquivalentInjection').reset_index().merge(boundary_points, + left_on="ID", + right_on='Terminal.ConductingEquipment', + suffixes=('_ConnectivityNode', '')) + + # Get paired injections at boundary points + paired_injections = injections.groupby("Terminal.ConnectivityNode").filter(lambda x: len(x) == 2) + + # Set terminal status + updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) + updated_terminal_status["KEY"] = "ACDCTerminal.connected" + updated_terminal_status["VALUE"] = "true" + + # Set Regulation off + updated_regulation_status = paired_injections[["ID"]].copy() + updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" + updated_regulation_status["VALUE"] = "false" + + # Set P to 0 + updated_p_value = paired_injections[["ID"]].copy() + updated_p_value["KEY"] = "EquivalentInjection.p" + updated_p_value["VALUE"] = 0 + + # Set Q to 0 + updated_q_value = paired_injections[["ID"]].copy() + updated_q_value["KEY"] = "EquivalentInjection.q" + updated_q_value["VALUE"] = 0 + + return data.update_triplet_from_triplet(pd.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) + + +def configure_paired_boundarypoint_injections_by_nodes(data): + # TODO [LEGACY] + """Where there are paired boundary points, equivalent injections need to be modified + Set P and Q to 0 - so that no additional consumption or production is on tie line + Set voltage control off - so that no additional consumption or production is on tie line + Set terminal to connected - to be sure we have paired connected injections at boundary point + + TODO NOTE THAT THIS IS COPY FROM 'configure_paired_boundarypoint_injections' + In some models terminals are missing references to ConnectivityNodes + """ + connectivity_boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] + topological_boundary_points = data.query("KEY == 'TopologicalNode.boundaryPoint' and VALUE == 'true'")[["ID"]] + try: + terminals = data.type_tableview("Terminal").reset_index()[['ID', + 'Terminal.ConductingEquipment', + 'Terminal.ConnectivityNode', + 'Terminal.TopologicalNode']] + except KeyError: + terminals = data.type_tableview("Terminal").reset_index()[['ID', + 'Terminal.ConductingEquipment', + 'Terminal.TopologicalNode']] + injections = data.type_tableview('EquivalentInjection').reset_index()[['ID', + # 'EquivalentInjection.p', + # 'EquivalentInjection.q', + # 'EquivalentInjection.regulationStatus' + ]] + topological_boundary_points = topological_boundary_points.merge(terminals, + left_on="ID", + right_on="Terminal.TopologicalNode", + suffixes=('_TopologicalNode', '_Terminal')) + topological_injections = injections.merge(topological_boundary_points, + left_on="ID", + right_on='Terminal.ConductingEquipment', + suffixes=('_ConnectivityNode', '')) + paired_topological_injections = (topological_injections.groupby("Terminal.TopologicalNode") + .filter(lambda x: len(x) == 2)) + paired_injections = paired_topological_injections + if 'Terminal.ConnectivityNode' in terminals: + connectivity_boundary_points = connectivity_boundary_points.merge(terminals, + left_on="ID", + right_on="Terminal.ConnectivityNode", + suffixes=('_ConnectivityNode', '_Terminal')) + connectivity_injections = injections.merge(connectivity_boundary_points, + left_on="ID", + right_on='Terminal.ConductingEquipment', + suffixes=('_TopologicalNode', '')) + + paired_connectivity_injections = (connectivity_injections.groupby("Terminal.ConnectivityNode") + .filter(lambda x: len(x) == 2)) + merged_injections = paired_connectivity_injections.merge(paired_topological_injections, + on='ID', + how='outer', + indicator=True, + suffixes=('_CN', '_TN')) + only_connectivity_injections = merged_injections[merged_injections['_merge'] == 'left_only'] + only_topological_injections = merged_injections[merged_injections['_merge'] == 'right_only'] + if len(only_connectivity_injections.index) != 0 or len(only_topological_injections.index) == 0: + paired_injections = paired_connectivity_injections + else: + logger.warning(f"Mismatch of finding paired injections from topological nodes and connectivity nodes") + else: + logger.warning(f"Terminals do not contain Connectivity nodes") + # Set terminal status + updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) + updated_terminal_status["KEY"] = "ACDCTerminal.connected" + updated_terminal_status["VALUE"] = "true" + + # Set Regulation off + updated_regulation_status = paired_injections[["ID"]].copy() + updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" + updated_regulation_status["VALUE"] = "false" + + # Set P to 0 + updated_p_value = paired_injections[["ID"]].copy() + updated_p_value["KEY"] = "EquivalentInjection.p" + updated_p_value["VALUE"] = 0 + + # Set Q to 0 + updated_q_value = paired_injections[["ID"]].copy() + updated_q_value["KEY"] = "EquivalentInjection.q" + updated_q_value["VALUE"] = 0 + + return data.update_triplet_from_triplet(pd.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) \ No newline at end of file diff --git a/emf/model_merger/temporary_fixes.py b/emf/model_merger/temporary_fixes.py deleted file mode 100644 index 897faeda..00000000 --- a/emf/model_merger/temporary_fixes.py +++ /dev/null @@ -1,462 +0,0 @@ -import pypowsybl -import triplets -import pandas as pd -import logging - -from emf.common.integrations import elastic -from emf.common.loadflow_tool.helper import get_model_outages, get_network_elements, get_opdm_data_from_models, load_opdm_data -from emf.model_merger.merge_functions import (create_sv_and_updated_ssh, fix_sv_shunts, - fix_sv_tapsteps, remove_duplicate_sv_voltages, - remove_small_islands, check_and_fix_dependencies, - configure_paired_boundarypoint_injections_by_nodes, - ) - - -logger = logging.getLogger(__name__) - - -def open_switches_in_network(network_pre_instance: pypowsybl.network.Network, switches_dataframe: pd.DataFrame): - """ - Opens switches in loaded network given bu dataframe (uses ID for merging) - :param network_pre_instance: pypowsybl Network instance where igms are loaded in - :param switches_dataframe: dataframe - """ - logger.info(f"Opening {len(switches_dataframe.index)} switches") - switches = network_pre_instance.get_switches(all_attributes=True).reset_index() - switches = switches.merge(switches_dataframe[['ID']].rename(columns={'ID': 'id'}), on='id') - non_retained_closed = switches.merge(switches_dataframe.rename(columns={'ID': 'id'}), - on='id')[['id', 'open']] - non_retained_closed['open'] = True - network_pre_instance.update_switches(non_retained_closed.set_index('id')) - return network_pre_instance - - -def check_net_interchanges(cgm_sv_data, cgm_ssh_data, original_models, fix_errors: bool = False, - threshold: float = None): - """ - An attempt to calculate the net interchange 2 values and check them against those provided in ssh profiles - :param cgm_sv_data: merged sv profile - :param cgm_ssh_data: merged ssh profile - :param original_models: original profiles - :param fix_errors: injects new calculated flows into merged ssh profiles - :param threshold: specify threshold if needed - :return (updated) ssh profiles - """ - original_models = get_opdm_data_from_models(model_data=original_models) - try: - control_areas = (original_models.type_tableview('ControlArea') - .rename_axis('ControlArea') - .reset_index())[['ControlArea', 'ControlArea.netInterchange', 'ControlArea.pTolerance', - 'IdentifiedObject.energyIdentCodeEic', 'IdentifiedObject.name']] - except KeyError: - control_areas = original_models.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() - ssh_areas = cgm_ssh_data.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() - control_areas = control_areas.merge(ssh_areas, on='ControlArea')[['ControlArea', 'ControlArea.netInterchange', - 'ControlArea.pTolerance', - 'IdentifiedObject.energyIdentCodeEic', - 'IdentifiedObject.name']] - tie_flows = (original_models.type_tableview('TieFlow') - .rename_axis('TieFlow').rename(columns={'TieFlow.ControlArea': 'ControlArea', - 'TieFlow.Terminal': 'Terminal'}) - .reset_index())[['ControlArea', 'Terminal', 'TieFlow.positiveFlowIn']] - tie_flows = tie_flows.merge(control_areas[['ControlArea']], on='ControlArea') - try: - terminals = (original_models.type_tableview('Terminal') - .rename_axis('Terminal').reset_index())[['Terminal', 'ACDCTerminal.connected']] - except KeyError: - terminals = (original_models.type_tableview('Terminal') - .rename_axis('Terminal').reset_index())[['Terminal']] - tie_flows = tie_flows.merge(terminals, on='Terminal') - try: - power_flows_pre = (original_models.type_tableview('SvPowerFlow') - .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) - .reset_index())[['Terminal', 'SvPowerFlow.p']] - tie_flows = tie_flows.merge(power_flows_pre, on='Terminal', how='left') - except Exception: - logger.error(f"Was not able to get tie flows from original models") - power_flows_post = (cgm_sv_data.type_tableview('SvPowerFlow') - .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) - .reset_index())[['Terminal', 'SvPowerFlow.p']] - - tie_flows = tie_flows.merge(power_flows_post, on='Terminal', how='left', - suffixes=('_pre', '_post')) - try: - tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p_pre', 'SvPowerFlow.p_post']] - .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) - .rename_axis('ControlArea').reset_index()) - except KeyError: - tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p']] - .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) - .rename_axis('ControlArea').reset_index()) - tie_flows_grouped = tie_flows_grouped.rename(columns={'SvPowerFlow.p': 'SvPowerFlow.p_post'}) - tie_flows_grouped = control_areas.merge(tie_flows_grouped, on='ControlArea') - if threshold and threshold > 0: - tie_flows_grouped['Exceeded'] = (abs(tie_flows_grouped['ControlArea.netInterchange'] - - tie_flows_grouped['SvPowerFlow.p_post']) > threshold) - else: - tie_flows_grouped['Exceeded'] = (abs(tie_flows_grouped['ControlArea.netInterchange'] - - tie_flows_grouped['SvPowerFlow.p_post']) > - tie_flows_grouped['ControlArea.pTolerance']) - net_interchange_errors = tie_flows_grouped[tie_flows_grouped.eval('Exceeded')] - if not net_interchange_errors.empty: - if threshold > 0: - logger.warning(f"Found {len(net_interchange_errors.index)} possible net interchange_2 problems " - f"over {threshold}") - else: - logger.warning(f"Found {len(net_interchange_errors.index)} possible net interchange_2 problems") - # print(net_interchange_errors.to_string()) - if fix_errors: - logger.warning(f"Updating {len(net_interchange_errors.index)} interchanges to new values") - new_areas = cgm_ssh_data.type_tableview('ControlArea').reset_index()[['ID', - 'ControlArea.pTolerance', 'Type']] - new_areas = new_areas.merge(net_interchange_errors[['ControlArea', 'SvPowerFlow.p_post']] - .rename(columns={'ControlArea': 'ID', - 'SvPowerFlow.p_post': 'ControlArea.netInterchange'}), on='ID') - cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(cgm_ssh_data, new_areas) - return cgm_ssh_data - - -def check_non_boundary_equivalent_injections(cgm_sv_data, - cgm_ssh_data, - original_models, - threshold: float = 0, - fix_errors: bool = False): - """ - Checks equivalent injections that are not on boundary topological nodes - :param cgm_sv_data: merged SV profile - :param cgm_ssh_data: merged SSH profile - :param original_models: igms - :param threshold: threshold for checking - :param fix_errors: if true then copies values from sv profile to ssh profile - :return cgm_ssh_data - """ - original_models = get_opdm_data_from_models(model_data=original_models) - boundary_nodes = original_models.query('KEY == "TopologicalNode.boundaryPoint" & VALUE == "true"')[['ID']] - terminals = (original_models.type_tableview('Terminal').rename_axis('SvPowerFlow.Terminal').reset_index() - .merge(boundary_nodes.rename(columns={'ID': 'Terminal.TopologicalNode'}), - on='Terminal.TopologicalNode', how='outer', indicator=True))[['SvPowerFlow.Terminal', - 'Terminal.ConductingEquipment', - '_merge']] - terminals = terminals[terminals['_merge'] == 'left_only'][['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] - return check_all_kind_of_injections(cgm_sv_data=cgm_sv_data, - cgm_ssh_data=cgm_ssh_data, - original_models=original_models, - injection_name='EquivalentInjection', - fields_to_check={'SvPowerFlow.p': 'EquivalentInjection.p'}, - threshold=threshold, - terminals=terminals, - fix_errors=fix_errors) - - -def check_all_kind_of_injections(cgm_sv_data, - cgm_ssh_data, - original_models, - injection_name: str = 'ExternalNetworkInjection', - fields_to_check: dict = None, - fix_errors: bool = False, - threshold: float = 0, - terminals: pd.DataFrame = None, - report_sum: bool = True): - """ - Compares the given cgm ssh injection values to the corresponding sv powerflow values in cgm sv - :param cgm_sv_data: merged SV profile - :param cgm_ssh_data: merged SSH profile - :param original_models: igms - :param injection_name: name of the injection - :param fields_to_check: dictionary where key is the field in powerflow and value is the field in injection - :param fix_errors: if true then copies values from sv profile to ssh profile - :param threshold: max allowed mismatch - :param terminals: optional, can give dataframe of terminals as input - :param report_sum: if true prints sum of injections and powerflows to console - :return cgm_ssh_data - """ - if not fields_to_check: - return cgm_ssh_data - fixed_fields = ['ID'] - original_models = get_opdm_data_from_models(model_data=original_models) - try: - original_injections = original_models.type_tableview(injection_name).reset_index() - injections = cgm_ssh_data.type_tableview(injection_name).reset_index() - except AttributeError: - logger.info(f"SSH profile doesn't contain data about {injection_name}") - return cgm_ssh_data - injections_reduced = injections[[*fixed_fields, *fields_to_check.values()]] - original_injections_reduced = original_injections[[*fixed_fields, *fields_to_check.values()]] - injections_reduced = injections_reduced.merge(original_injections_reduced, on='ID', suffixes=('', '_org')) - if terminals is None: - terminals = (original_models.type_tableview('Terminal') - .rename_axis('SvPowerFlow.Terminal') - .reset_index())[['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] - flows = (cgm_sv_data.type_tableview('SvPowerFlow') - .reset_index())[[*['SvPowerFlow.Terminal'], *fields_to_check.keys()]] - terminals = terminals.merge(flows, on='SvPowerFlow.Terminal') - terminals = terminals.merge(injections_reduced, left_on='Terminal.ConductingEquipment', right_on='ID') - - filtered_list = [] - for flow_field, injection_field in fields_to_check.items(): - filtered_list.append(terminals[abs(terminals[injection_field] - terminals[flow_field]) > threshold]) - if report_sum: - logger.info(f"IGM {injection_field} = {terminals[injection_field + '_org'].sum()} vs " - f"CGM {injection_field} = {terminals[injection_field].sum()} vs " - f"CGM {flow_field} = {terminals[flow_field].sum()}") - if not filtered_list: - return cgm_ssh_data - filtered = pd.concat(filtered_list).drop_duplicates().reset_index(drop=True) - if not filtered.empty: - logger.warning(f"Found {len(filtered.index)} mismatches between {injection_name} and flow values on terminals") - if fix_errors: - logger.info(f"Updating {injection_name} values from terminal flow values") - injections_update = injections.merge(filtered[[*fixed_fields, *fields_to_check.keys()]]) - injections_update = injections_update.drop(columns=fields_to_check.values()) - injections_update = injections_update.rename(columns=fields_to_check) - cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(data=cgm_ssh_data, - tableview=injections_update, - update=True, - add=False) - return cgm_ssh_data - - -def run_post_merge_processing(input_models: list, - merged_model: object, - task_properties: dict, - small_island_size: str, - enable_temp_fixes: bool, - time_horizon: str | None = None): - - time_horizon = time_horizon or task_properties["time_horizon"] - scenario_datetime = task_properties["timestamp_utc"] - merging_area = task_properties["merge_type"] - merging_entity = task_properties["merging_entity"] - mas = task_properties["mas"] - version = task_properties["version"] - - models_as_triplets = load_opdm_data(input_models) - sv_data, ssh_data, opdm_object_meta = create_sv_and_updated_ssh(merged_model, input_models, models_as_triplets, - scenario_datetime, time_horizon, - version, merging_area, - merging_entity, mas) - fix_net_interchange_errors = task_properties.get('fix_net_interchange2', False) - net_interchange_threshold = int(task_properties.get('net_interchange2_threshold', 200)) - injection_threshold = 0.1 - fix_injection_errors = True - if enable_temp_fixes: - sv_data = fix_sv_shunts(sv_data, models_as_triplets) - sv_data = fix_sv_tapsteps(sv_data, ssh_data) - sv_data = remove_small_islands(sv_data, int(small_island_size)) - sv_data = remove_duplicate_sv_voltages(cgm_sv_data=sv_data, original_data=models_as_triplets) - sv_data = check_and_fix_dependencies(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=models_as_triplets) - #sv_data, ssh_data = disconnect_equipment_if_flow_sum_not_zero(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=models_as_triplets) fix implemented in pypowsybl 1.8.1 - ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, - cgm_sv_data=sv_data, - original_models=models_as_triplets, - injection_name='EnergySource', - threshold=injection_threshold, - fields_to_check={'SvPowerFlow.p': 'EnergySource.activePower'}, - fix_errors=fix_injection_errors) - ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, - cgm_sv_data=sv_data, - original_models=models_as_triplets, - injection_name='ExternalNetworkInjection', - fields_to_check={'SvPowerFlow.p': 'ExternalNetworkInjection.p'}, - threshold=injection_threshold, - fix_errors=fix_injection_errors) - ssh_data = check_non_boundary_equivalent_injections(cgm_sv_data=sv_data, - cgm_ssh_data=ssh_data, - original_models=models_as_triplets, - threshold=injection_threshold, - fix_errors=fix_injection_errors) - try: - ssh_data = check_net_interchanges(cgm_sv_data=sv_data, - cgm_ssh_data=ssh_data, - original_models=models_as_triplets, - fix_errors=fix_net_interchange_errors, - threshold=net_interchange_threshold) - except KeyError: - logger.warning(f"No fields for netInterchange") - - return sv_data, ssh_data, opdm_object_meta - - -def fix_model_outages(merged_model: object, tso_list: list, scenario_datetime: str, time_horizon: str, debug: bool = False): - - area_map = {"LITGRID": "Lithuania", "AST": "Latvia", "ELERING": "Estonia"} - outage_areas = [area_map.get(item, item) for item in tso_list] - - elk_service = elastic.Elastic() - - # Get outage eic-mrid mapping - mrid_map = elk_service.get_docs_by_query(index='config-network*', query={"match_all": {}}, size=10000) - mrid_map['mrid'] = mrid_map['mrid'].str.lstrip('_') - - # Get latest UAP parse date - if time_horizon == 'MO': - merge_type = "Month" - elif time_horizon == 'YR': - merge_type = "Year" - else: - merge_type = "Week" - - body = {"size": 1, "query": {"bool": {"must": [{"match": {"Merge": merge_type}}]}}, - "sort": [{"reportParsedDate": {"order": "desc"}}], "fields": ["reportParsedDate"]} - last_uap_version = elk_service.client.search(index='opc-outages-baltics*', body=body)['hits']['hits'][0]['fields']['reportParsedDate'][0] - - # Query for latest outage UAP - uap_query = {"bool": {"must": [{"match": {"reportParsedDate": f"{last_uap_version}"}}, - {"match": {"Merge": merge_type}}]}} - uap_outages = elk_service.get_docs_by_query(index='opc-outages-baltics*', query=uap_query, size=10000) - uap_outages = uap_outages.merge(mrid_map[['eic', 'mrid']], how='left', on='eic', indicator=True).rename(columns={"mrid": 'grid_id'}) - unmapped_outages = uap_outages[uap_outages['_merge'] == 'left_only'] - - if not unmapped_outages.empty: - logger.error(f"Unable to map these outage mRIDs: {unmapped_outages['name'].values}") - - # Filter outages according to model scenario date and replaced area - filtered_outages = uap_outages[(uap_outages['start_date'] <= scenario_datetime) & (uap_outages['end_date'] >= scenario_datetime)] - filtered_outages = filtered_outages[filtered_outages['Area'].isin(outage_areas)] - mapped_outages = filtered_outages[~filtered_outages['grid_id'].isna()] - - # Get disconnected elements in network model - model_outages = pd.DataFrame(get_model_outages(network=merged_model.network)) - mapped_model_outages = pd.merge(model_outages, mrid_map, left_on='grid_id', right_on='mrid', how='inner') - model_area_map = {"LITGRID": "LT", "AST": "LV", "ELERING": "EE"} - model_outage_areas = [model_area_map.get(item, item) for item in tso_list] - filtered_model_outages = mapped_model_outages[mapped_model_outages['country'].isin(model_outage_areas)] - - # Include cross-border lines for reconnection (both dangling lines) - dangling_lines = get_network_elements(network=merged_model.network, - element_type=pypowsybl.network.ElementType.DANGLING_LINE).reset_index(names=['grid_id']) - border_lines = dangling_lines[dangling_lines['pairing_key'].isin(model_outages['pairing_key'])] - relevant_border_lines = border_lines[border_lines['country'].isin(['LT', 'LV', 'EE'])] - additional_dangling_lines = dangling_lines[dangling_lines['pairing_key'].isin(relevant_border_lines['pairing_key'])] - - # Merged dataframe of network elements to be reconnected - filtered_model_outages = pd.concat([filtered_model_outages, additional_dangling_lines]).drop_duplicates(subset='grid_id') - filtered_model_outages = filtered_model_outages.where(pd.notnull(filtered_model_outages), None) - - # rename columns - filtered_model_outages = filtered_model_outages.copy()[['name', 'grid_id', 'eic']].rename(columns={'grid_id': 'mrid'}) - mapped_outages = mapped_outages.copy()[['name', 'grid_id', 'eic']].rename(columns={'grid_id': 'mrid'}) - - logger.info("Fixing outages inside merged model") - - # Reconnecting outages from network-config list - outages_updated = {} - for index, outage in filtered_model_outages.iterrows(): - try: - if merged_model.network.connect(outage['mrid']): - logger.info(f"Successfully reconnected: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages = True - outage_dict = outage.to_dict() - outage_dict.update({'status': 'connected'}) - outages_updated[outage_dict['mrid']] = outage_dict - else: - if uap_outages['grid_id'].str.contains(outage['mrid']).any(): - logger.info(f"Element is already connected: {outage['name']} [mrid: {outage['mrid']}]") - else: - logger.error(f"Failed to connect element: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - except Exception as e: - logger.error((e, outage['name'])) - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - merged_model.outages = False - continue - - # Applying outages from UAP - for index, outage in mapped_outages.iterrows(): - try: - if merged_model.network.disconnect(outage['mrid']): - logger.info(f"Successfully disconnected: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages = True - outage_dict = outage.to_dict() - outage_dict.update({'status': 'disconnected'}) - outages_updated[outage_dict['mrid']] = outage_dict - else: - if uap_outages['grid_id'].str.contains(outage['mrid']).any(): - logger.info(f"Element is already in outage: {outage['name']} [mrid: {outage['mrid']}]") - else: - logger.error(f"Failed to disconnect element: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - except Exception as e: - logger.error((e, outage['name'])) - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - merged_model.outages = False - continue - - # Keep only important keys of updated outages - merged_model.outages_updated = list(outages_updated.values()) - - if merged_model.outages_unmapped: - merged_model.outages = False - - return merged_model - - -def fix_igm_ssh_vs_cgm_ssh_error(network_pre_instance: pypowsybl.network.Network): - """ - Implements various fixes to suppress igm ssh vs cgm ssh error - 1) Get all generators and remove them from slack distribution - 2) If generators have target_p outside the endpoints ('limits') of a curve then set it to be within - 3) Condensers p should not be modified so if it is not 0 then it sets the target_p to equal the existing p - :param network_pre_instance: pypowsybl Network instance where igms are loaded in - :return updated network_pre_instance - """ - try: - all_generators = network_pre_instance.get_elements(element_type=pypowsybl.network.ElementType.GENERATOR, - all_attributes=True).reset_index() - generators_mask = (all_generators['CGMES.synchronousMachineOperatingMode'].str.contains('generator')) - not_generators = all_generators[~generators_mask] - generators = all_generators[generators_mask] - curve_points = (network_pre_instance - .get_elements(element_type=pypowsybl.network.ElementType.REACTIVE_CAPABILITY_CURVE_POINT, - all_attributes=True).reset_index()) - curve_limits = (curve_points.merge(generators[['id']], on='id') - .groupby('id').agg(curve_p_min=('p', 'min'), curve_p_max=('p', 'max'))).reset_index() - curve_generators = generators.merge(curve_limits, on='id') - # low end can be zero - curve_generators = curve_generators[(curve_generators['target_p'] > curve_generators['curve_p_max']) | - ((curve_generators['target_p'] > 0) & - (curve_generators['target_p'] < curve_generators['curve_p_min']))] - if not curve_generators.empty: - logger.warning(f"Found {len(curve_generators.index)} generators for " - f"which p > max(reactive capacity curve(p)) or p < min(reactive capacity curve(p))") - - # Solution 1: set max_p from curve max, it should contain p on target-p - upper_limit_violated = curve_generators[(curve_generators['max_p'] > curve_generators['curve_p_max'])] - if not upper_limit_violated.empty: - logger.warning(f"Updating max p from curve for {len(upper_limit_violated.index)} generators") - upper_limit_violated['max_p'] = upper_limit_violated['curve_p_max'] - network_pre_instance.update_generators(upper_limit_violated[['id', 'max_p']].set_index('id')) - - lower_limit_violated = curve_generators[(curve_generators['min_p'] < curve_generators['curve_p_min'])] - if not lower_limit_violated.empty: - logger.warning(f"Updating min p from curve for {len(lower_limit_violated.index)} generators") - lower_limit_violated['min_p'] = lower_limit_violated['curve_p_min'] - network_pre_instance.update_generators(lower_limit_violated[['id', 'min_p']].set_index('id')) - - # Solution 2: discard generator from participating - extensions = network_pre_instance.get_extensions('activePowerControl') - remove_curve_generators = extensions.merge(curve_generators[['id']], - left_index=True, right_on='id') - if not remove_curve_generators.empty: - remove_curve_generators['participate'] = False - network_pre_instance.update_extensions('activePowerControl', - remove_curve_generators.set_index('id')) - condensers = all_generators[(all_generators['CGMES.synchronousMachineType'].str.contains('condenser')) - & (abs(all_generators['p']) > 0) - & (abs(all_generators['target_p']) == 0)] - # Fix condensers that have p not zero by setting their target_p to equal to p - if not condensers.empty: - logger.warning(f"Found {len(condensers.index)} condensers for which p ~= 0 & target_p = 0") - condensers.loc[:, 'target_p'] = condensers['p'] * (-1) - network_pre_instance.update_generators(condensers[['id', 'target_p']].set_index('id')) - # Remove all not generators from active power distribution - if not not_generators.empty: - logger.warning(f"Removing {len(not_generators.index)} machines from power distribution") - extensions = network_pre_instance.get_extensions('activePowerControl') - remove_not_generators = extensions.merge(not_generators[['id']], left_index=True, right_on='id') - remove_not_generators['participate'] = False - remove_not_generators = remove_not_generators.set_index('id') - network_pre_instance.update_extensions('activePowerControl', remove_not_generators) - except Exception as ex: - logger.warning(f"Unable to pre-process for igm-cgm-ssh error: {ex}") - return network_pre_instance diff --git a/emf/model_retriever/model_retriever.py b/emf/model_retriever/model_retriever.py index 82ddbafb..71be14b9 100644 --- a/emf/model_retriever/model_retriever.py +++ b/emf/model_retriever/model_retriever.py @@ -3,8 +3,8 @@ from io import BytesIO import json import triplets - -from emf.common.loadflow_tool.helper import zip_xml, create_opdm_objects +from emf.common.helpers.opdm_objects import create_opdm_objects +from emf.common.helpers.utils import zip_xml from emf.common.config_parser import parse_app_properties from emf.common.integrations import elastic, opdm, minio_api @@ -43,7 +43,7 @@ def handle(self, message: bytes, properties: dict, **kwargs): # Repackage to form of zip(xml) rdfzip_files = [] for xml in rdfxml_files: - rdfzip_files.append(zip_xml(xml)) + rdfzip_files.append(zip_xml(xml_file_object=xml)) # Create OPDM objects opdm_objects = create_opdm_objects(models=[rdfzip_files], metadata={"data-source": "PDN"}) diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index e83d4eff..c5f6bd58 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -7,12 +7,14 @@ import pypowsybl as pp import uuid import triplets -from emf.common.loadflow_tool.helper import attr_to_dict, load_model from emf.common.config_parser import parse_app_properties -from emf.common.integrations import elastic, minio_api, rabbit +from emf.common.integrations import elastic, minio_api from emf.common.integrations.object_storage import models -from emf.common.loadflow_tool.helper import load_opdm_data, clean_data_from_opdm_objects, export_to_cgmes_zip from emf.common.loadflow_tool import loadflow_settings +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, clean_data_from_opdm_objects +from emf.common.helpers.loadflow import load_network_model +from emf.common.helpers.utils import attr_to_dict +from emf.common.helpers.cgmes import export_to_cgmes_zip from emf.model_validator import validator_functions from emf.common.decorators import performance_counter @@ -73,6 +75,7 @@ def validate_loadflow(self): # TODO currently storing only main island results main_component = loadflow_result[0] component_results = attr_to_dict(main_component) + logger.info(f"Loadflow status: {component_results['status_text']}") component_results['status'] = component_results.get('status').value component_results['distributed_active_power'] = 0.0 if math.isnan(component_results['distributed_active_power'])\ else component_results['distributed_active_power'] @@ -191,12 +194,12 @@ def handle(self, message: bytes, properties: dict, **kwargs): report = {} try: # Run pre-loadflow validations - network_triplets = load_opdm_data(opdm_objects=[opdm_object, latest_boundary]) + network_triplets = load_opdm_objects_to_triplets(opdm_objects=[opdm_object, latest_boundary]) pre_lf_validation = PreLFValidator(network=network_triplets) pre_lf_validation.run_validation() # Run post-loadflow validations - network = load_model(opdm_objects=[opdm_object, latest_boundary]) + network = load_network_model(opdm_objects=[opdm_object, latest_boundary]) post_lf_validation = PostLFValidator(network=network, network_triplets=network_triplets) post_lf_validation.run_validation() diff --git a/emf/model_validator/validator_functions.py b/emf/model_validator/validator_functions.py index 36302b37..0f7ecf64 100644 --- a/emf/model_validator/validator_functions.py +++ b/emf/model_validator/validator_functions.py @@ -1,7 +1,7 @@ import logging import pandas import triplets -from emf.common.loadflow_tool.helper import get_opdm_data_from_models, create_opdm_objects, export_to_cgmes_zip +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ def get_nodes_against_kirchhoff_first_law(original_models, :param nodes_only: if true then return unique nodes only, if false then nodes with corresponding terminals :param sv_injection_limit: threshold for deciding whether the node is violated by sum of flows """ - original_models = get_opdm_data_from_models(model_data=original_models) + original_models = load_opdm_objects_to_triplets(opdm_objects=original_models) sv_injections = pandas.DataFrame() if cgm_sv_data is None: cgm_sv_data = original_models @@ -79,7 +79,7 @@ def check_not_retained_switches_between_nodes(original_data, open_not_retained_s :return: updated original data """ updated_switches = 0 - original_models = get_opdm_data_from_models(original_data) + original_models = load_opdm_objects_to_triplets(opdm_objects=original_data) not_retained_switches = original_models[(original_models['KEY'] == 'Switch.retained') & (original_models['VALUE'] == "false")][['ID']] closed_switches = original_models[(original_models['KEY'] == 'Switch.open') @@ -118,7 +118,7 @@ def get_not_retained_switches_between_nodes(original_data): :return: updated original data """ updated_switches = False - original_models = get_opdm_data_from_models(original_data) + original_models = load_opdm_objects_to_triplets(opdm_objects=original_data) not_retained_switches = original_models[(original_models['KEY'] == 'Switch.retained') & (original_models['VALUE'] == "false")][['ID']] closed_switches = original_models[(original_models['KEY'] == 'Switch.open') diff --git a/emf/task_generator/manual_worker.py b/emf/task_generator/manual_worker.py index 98810c6c..bdee904d 100644 --- a/emf/task_generator/manual_worker.py +++ b/emf/task_generator/manual_worker.py @@ -2,7 +2,8 @@ import os import config import logging -from emf.task_generator.task_generator import generate_tasks, filter_and_flatten_dict +from emf.task_generator.task_generator import generate_tasks +from emf.common.helpers.utils import filter_and_flatten_dict from emf.common.integrations import rabbit from emf.common.config_parser import parse_app_properties from emf.common.logging.custom_logger import initialize_custom_logger diff --git a/emf/task_generator/task_generator.py b/emf/task_generator/task_generator.py index 27fd9a40..52fbd313 100644 --- a/emf/task_generator/task_generator.py +++ b/emf/task_generator/task_generator.py @@ -6,7 +6,8 @@ import json import logging from os import getlogin -from emf.common.time_helper import parse_duration, convert_to_utc, convert_to_timezone, timezone, reference_times, utcnow +from emf.common.helpers.time import parse_duration, convert_to_utc, convert_to_timezone, timezone, reference_times, utcnow +from emf.common.helpers.tasks import update_task_status from emf.common.integrations.elastic import Elastic from emf.common.config_parser import parse_app_properties @@ -187,75 +188,6 @@ def generate_tasks(task_window_duration: str, run_timestamp = runs.get_next(datetime) -def flatten_dict(nested_dict: dict, parent_key: str = '', separator: str = '.'): - """ - Flattens a nested dictionary. - - Parameters: - - nested_dict (dict): The dictionary to flatten. - - parent_key (str): The base key string used for recursion. - - separator (str): The separator between parent and child keys. - - Returns: - - dict: A flattened dictionary where nested keys are concatenated into a single string. - """ - items = [] - for k, v in nested_dict.items(): - new_key = f"{parent_key}{separator}{k}" if parent_key else k - if isinstance(v, dict): - items.extend(flatten_dict(v, new_key, separator=separator).items()) - elif isinstance(v, list): - for i, item in enumerate(v): - if isinstance(item, dict): - items.extend(flatten_dict(item, f"{new_key}[{i}]", separator=separator).items()) - else: - items.append((f"{new_key}[{i}]", item)) - else: - items.append((new_key, v)) - return dict(items) - - -def filter_and_flatten_dict(nested_dict: dict, keys: list): - """ - Creates a new flat dictionary from specified keys. - - Parameters: - - nested_dict (dict): The original nested dictionary. - - keys (list): The list of keys to include in the new flat dictionary. - - Returns: - - dict: A new flat dictionary with only the specified keys. - """ - flattened = flatten_dict(nested_dict) - return {key: flattened[key] for key in keys if key in flattened} - - -def update_task_status(task: dict, status_text: str, publish: bool = True): - """Update task status - Will update task_update_time - Will update task_status - Will append new status to task_status_trace""" - - logger.info(f"Updating Task status to: {status_text}") - - utc_now = datetime.utcnow().isoformat() - - task["task_update_time"] = utc_now - task["task_status"] = status_text - - task["task_status_trace"].append({ - "status": status_text, - "timestamp": utc_now - }) - - # TODO - better handling if elk is not available, possibly set elk connection timeout really small or refactor the sending to happen via rabbit - if publish: - try: - Elastic.send_to_elastic(index=TASK_ELK_INDEX, json_message=task, id=task.get("@id")) - except Exception as e: - logger.error(f"Task publication to Elastic failed with error: {e}") - - def set_task_version(task: dict): # Check versioning mode diff --git a/emf/task_generator/worker.py b/emf/task_generator/worker.py index 72510e5b..c4789e32 100644 --- a/emf/task_generator/worker.py +++ b/emf/task_generator/worker.py @@ -1,7 +1,8 @@ import json import config import logging -from emf.task_generator.task_generator import generate_tasks, filter_and_flatten_dict +from emf.task_generator.task_generator import generate_tasks +from emf.common.helpers.utils import filter_and_flatten_dict from emf.common.integrations import rabbit from emf.common.config_parser import parse_app_properties from emf.common.logging.custom_logger import initialize_custom_logger From 47411ff031ef36c7afb33cfbe4162a4b80ab1ac9 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Thu, 26 Jun 2025 10:45:08 +0300 Subject: [PATCH 22/67] logging: clarified object storage log message about source bucket --- emf/common/integrations/object_storage/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emf/common/integrations/object_storage/models.py b/emf/common/integrations/object_storage/models.py index 77f77e19..2d63b4c1 100644 --- a/emf/common/integrations/object_storage/models.py +++ b/emf/common/integrations/object_storage/models.py @@ -166,7 +166,7 @@ def get_content(metadata: dict): components_received = [] for component in metadata["opde:Component"]: content_reference = component.get("opdm:Profile").get("pmd:content-reference") - logger.info(f"Downloading object: {content_reference}") + logger.info(f"Downloading object: {bucket_name}/{content_reference}") content = object_storage.minio_service.download_object(bucket_name, content_reference) component["opdm:Profile"]["DATA"] = content components_received.append(bool(content)) # collect boolean flags of received components From 476e1733026fb53e00a2c9a48c7f8dec8c030b0a Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Thu, 26 Jun 2025 11:47:42 +0300 Subject: [PATCH 23/67] bugfix: model validator fix to sanitize minio URL for EQ profile --- emf/model_validator/model_validator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index c5f6bd58..dc69ba04 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -213,7 +213,7 @@ def handle(self, message: bytes, properties: dict, **kwargs): for component in opdm_object['opde:Component']: # Map exported modified file with initial inside opdm object cgmes_file = [i for i in cgmes_modified_model if i.name == component['opdm:Profile']['pmd:fileName']][0] - cgmes_file.name = component['opdm:Profile']['pmd:content-reference'] + cgmes_file.name = component['opdm:Profile']['pmd:content-reference'].replace('//', '/') logger.info(f"Uploading modified model content to Minio: {cgmes_file.name}") self.minio_service.upload_object(file_path_or_file_object=cgmes_file, bucket_name=opdm_object['minio-bucket'], @@ -231,7 +231,7 @@ def handle(self, message: bytes, properties: dict, **kwargs): report['content_reference'] = opdm_object['pmd:content-reference'] report['tso'] = opdm_object['pmd:TSO'] report['duration_s'] = round(time.time() - start_time, 3) - report['minio-bucket'] = opdm_object['minio-bucket'] + report['minio_bucket'] = opdm_object['minio-bucket'] except Exception as error: logger.error(f"Models validator failed with exception: {error}", exc_info=True) From cd192e556645362652186a8c12ef1456d80dd2c0 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Thu, 26 Jun 2025 15:26:48 +0300 Subject: [PATCH 24/67] #337 scaling: implemented feature to keep power factor constant at DC scaling and total island net position alignment --- emf/model_merger/scaler.py | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/emf/model_merger/scaler.py b/emf/model_merger/scaler.py index dd4f8241..2b8199d0 100644 --- a/emf/model_merger/scaler.py +++ b/emf/model_merger/scaler.py @@ -22,6 +22,13 @@ scheduling areas are below the discrepancy thresholds, as defined previously; " In any case after the 15th iteration16 (adjustments take place within the iterations). """ + +""" +NOTES: + - power factor sign defines whether P and Q values has opposite sign. This needs to be ensured because new Q values +are calculated from P values, then the power factors sign defines what sign should be for new Q value. +""" + import pypowsybl as pp import logging import pandas as pd @@ -110,15 +117,20 @@ def scale_balance(model: object, # Get pre-scale HVDC setpoints dangling_lines = get_network_elements(network, pp.network.ElementType.DANGLING_LINE, all_attributes=True) + dangling_lines['power_factor'] = dangling_lines['boundary_q'] / dangling_lines['boundary_p'] # estimate the power factor dangling_lines['boundary_p'] = dangling_lines['boundary_p'] * -1 # invert boundary_p sign to match flow direction - prescale_hvdc_sp = dangling_lines[dangling_lines.isHvdc == 'true'][['lineEnergyIdentificationCodeEIC', 'boundary_p']] - prescale_hvdc_sp = prescale_hvdc_sp.rename(columns={'boundary_p': 'value'}) + dangling_lines['boundary_q'] = dangling_lines['boundary_q'] * -1 # invert boundary_q sign to match flow direction + prescale_hvdc_sp = dangling_lines[dangling_lines.isHvdc == 'true'][['lineEnergyIdentificationCodeEIC', 'boundary_p', 'boundary_q']] + prescale_hvdc_sp = prescale_hvdc_sp.rename(columns={'boundary_p': 'value', 'boundary_q': 'value_q'}) _hvdc_results.append(pd.concat([prescale_hvdc_sp.set_index('lineEnergyIdentificationCodeEIC').value, pd.Series({'KEY': 'prescale-setpoint'})]).to_dict()) - for dclink in prescale_hvdc_sp.to_dict('records'): - logger.info(f"[INITIAL] PRE-SCALE HVDC setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value'], 2)} MW") + for dclink in prescale_hvdc_sp.sort_values('lineEnergyIdentificationCodeEIC').to_dict('records'): + logger.info(f"[INITIAL] PRE-SCALE HVDC active power setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value'], 2)} MW") + logger.debug(f"[INITIAL] PRE-SCALE HVDC reactive power setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value_q'], 2)} MW") + # Mapping HVDC schedules to network - scalable_hvdc = dangling_lines[dangling_lines.isHvdc == 'true'][['lineEnergyIdentificationCodeEIC', _country_col, 'ucte_xnode_code']] + _cols_to_keep = ['lineEnergyIdentificationCodeEIC', _country_col, 'ucte_xnode_code', 'power_factor'] + scalable_hvdc = dangling_lines[dangling_lines.isHvdc == 'true'][_cols_to_keep] scalable_hvdc.reset_index(inplace=True) scalable_hvdc = scalable_hvdc.merge(target_hvdc_sp_df, left_on='lineEnergyIdentificationCodeEIC', right_on='registered_resource') mask = (scalable_hvdc[_country_col] == scalable_hvdc['in_domain']) | (scalable_hvdc[_country_col] == scalable_hvdc['out_domain']) @@ -132,13 +144,15 @@ def scale_balance(model: object, scalable_hvdc = scalable_hvdc.set_index('id') # Updating HVDC network elements to scheduled values - scalable_hvdc_target = scalable_hvdc[['value', 'lineEnergyIdentificationCodeEIC']] - network.update_dangling_lines(id=scalable_hvdc_target.index, p0=scalable_hvdc_target.value) + scalable_hvdc_target = scalable_hvdc[['value', 'lineEnergyIdentificationCodeEIC', 'power_factor']] + scalable_hvdc_target['value_q'] = scalable_hvdc_target.value * scalable_hvdc_target.power_factor # ensure power factor is kept + network.update_dangling_lines(id=scalable_hvdc_target.index, p0=scalable_hvdc_target.value, q0=scalable_hvdc_target.value_q) _hvdc_results.append(pd.concat([scalable_hvdc_target.set_index('lineEnergyIdentificationCodeEIC').value, pd.Series({'KEY': 'postscale-setpoint'})]).to_dict()) logger.info(f"[INITIAL] HVDC elements updated to target values: {scalable_hvdc_target['lineEnergyIdentificationCodeEIC'].values}") - for dclink in scalable_hvdc_target.to_dict('records'): - logger.info(f"[INITIAL] POST-SCALE HVDC setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value'], 2)} MW") + for dclink in scalable_hvdc_target.sort_values('lineEnergyIdentificationCodeEIC').to_dict('records'): + logger.info(f"[INITIAL] POST-SCALE HVDC active power setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value'], 2)} MW") + logger.debug(f"[INITIAL] POST-SCALE HVDC reactive power setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value_q'], 2)} MVar") # Get AC net positions scaling perimeter -> non-negative ConformLoads loads = get_network_elements(network, pp.network.ElementType.LOAD, all_attributes=True) @@ -194,6 +208,8 @@ def scale_balance(model: object, # Get pre-scale total network balance by each component -> AC+DC net position dangling_lines = get_network_elements(network, pp.network.ElementType.DANGLING_LINE, all_attributes=True) + dangling_lines['power_factor'] = dangling_lines['boundary_q'] / dangling_lines['boundary_p'] # estimate the power + dangling_lines['power_factor'] = dangling_lines['power_factor'].fillna(0) # handle zero division dangling_lines['boundary_p'] = dangling_lines['boundary_p'] * -1 # invert boundary_p sign to match flow direction prescale_network_np = {k: round(dangling_lines[dangling_lines.country.isin(v['countries'])].boundary_p.sum()) for k, v in converged_components.items()} _scaling_results.append({'KEY': 'prescale-network-np', 'GLOBAL': prescale_network_np, 'ITER': _iteration}) @@ -221,7 +237,8 @@ def scale_balance(model: object, prescale_network_acnp_target.dropna(inplace=True) logger.info(f"[ITER {_iteration}] Scaling network component {component_key} {v['countries']} ACNP to scheduled: {scheduled_component_acnp}") network.update_dangling_lines(id=prescale_network_acnp_target.index, - p0=prescale_network_acnp_target.to_list()) # TODO maintain power factor + p0=prescale_network_acnp_target.to_list(), + q0=(prescale_network_acnp_target * relevant_dangling_lines.power_factor).to_list()) _scaling_results.append({'KEY': 'target-network-acnp', 'GLOBAL': target_network_acnp, 'ITER': _iteration}) # Solving loadflow after aligning total network AC net position to scheduled From 5769916a038352512835ae16efd88ab497a41993 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Thu, 26 Jun 2025 15:29:55 +0300 Subject: [PATCH 25/67] bugfix: local replacement overwritten other additional received models --- emf/model_merger/model_merger.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 2fcf719c..607c995d 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -184,7 +184,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): filter_on='pmd:TSO') missing_local_import = [tso for tso in local_import_models if tso not in [model['pmd:TSO'] for model in additional_models_data]] - merged_model.excluded.extend([{'tso': tso, 'reason': 'Missing in PDN'} for tso in missing_local_import]) + merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-pdn'} for tso in missing_local_import]) # Perform local replacement if configured if model_replacement_local and missing_local_import: @@ -200,8 +200,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): 'time_horizon': model['pmd:timeHorizon'], 'scenario_timestamp': model['pmd:scenarioDate']} for model in replacement_models_local] merged_model.replaced_entity.extend(replaced_entities_local) - #TODO change, keeping this to keep consistent naming structure for now - additional_models_data = replacement_models_local + additional_models_data.extend(replacement_models_local) except Exception as error: logger.error(f"Failed to run replacement: {error} {error.with_traceback()}") else: From d371b34d2e10bc7666356f197005a708247ca2b5 Mon Sep 17 00:00:00 2001 From: mr0321 <145195994+mr0321@users.noreply.github.com> Date: Fri, 27 Jun 2025 11:10:22 +0300 Subject: [PATCH 26/67] moves preprocessing step for opening non-retained switches from merger to validator. Addresses #302 --- .../model_validator.properties | 3 +- emf/model_validator/model_validator.py | 18 ++++++++--- emf/model_validator/validator_functions.py | 31 ------------------- 3 files changed, 16 insertions(+), 36 deletions(-) diff --git a/config/model_validator/model_validator.properties b/config/model_validator/model_validator.properties index 3ec7c308..0c21164e 100644 --- a/config/model_validator/model_validator.properties +++ b/config/model_validator/model_validator.properties @@ -7,4 +7,5 @@ ELK_ID_FROM_METADATA_FIELDS = opde:Id,data-source ELK_ID_HASHING = True VALIDATION_LOAD_FLOW_SETTINGS = CGM_RELAXED_2 CHECK_NON_RETAINED_SWITCHES = False -CHECK_KIRCHHOFF_FIRST_LAW = False \ No newline at end of file +CHECK_KIRCHHOFF_FIRST_LAW = False +OPEN_NON_RETAINED_SWITCHES = False \ No newline at end of file diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index dc69ba04..1e26deee 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -29,11 +29,17 @@ def __init__(self, network: pd.DataFrame): self.network = network self.report = {'pre_validations': {}} - def validate_non_retained_switches(self): + def validate_non_retained_switches(self, open_non_retained_switches: bool = False): # TODO Use carefully as this might change switch statuses in the original model # TODO Currently disabled returning modified data non_retained_switched_valid = True - non_retained_switches = validator_functions.check_not_retained_switches_between_nodes(self.network)[1] + if open_non_retained_switches: + self.network, non_retained_switches = ( + validator_functions + .check_not_retained_switches_between_nodes(original_data=self.network, + open_not_retained_switches=open_non_retained_switches)) + else: + non_retained_switches = validator_functions.check_not_retained_switches_between_nodes(self.network)[1] if non_retained_switches: non_retained_switched_valid = False self.report['pre_validations']['non_retained_switches'] = non_retained_switched_valid @@ -44,8 +50,12 @@ def validate_kirchhoff_first_law(self): self.report['pre_validations']['kirchhoff_first_law'] = kirchhoff_first_law_valid def run_validation(self): + try: + open_non_retained_switches = json.loads(OPEN_NON_RETAINED_SWITCHES.lower()) + except (json.decoder.JSONDecodeError, AttributeError): + open_non_retained_switches = False if json.loads(CHECK_NON_RETAINED_SWITCHES.lower()): - self.validate_non_retained_switches() + self.validate_non_retained_switches(open_non_retained_switches=open_non_retained_switches) if json.loads(CHECK_KIRCHHOFF_FIRST_LAW.lower()): self.validate_kirchhoff_first_law() @@ -297,7 +307,7 @@ def handle(self, message: bytes, properties: dict, **kwargs): # Validate models for model in available_models: - network_triplets = load_opdm_data(opdm_objects=[model, latest_boundary]) + network_triplets = load_opdm_objects_to_triplets(opdm_objects=[model, latest_boundary]) network = load_model(opdm_objects=[model, latest_boundary]) post_lf_validation = PostLFValidator(network=network, network_triplets=network_triplets) post_lf_validation.run_validation() diff --git a/emf/model_validator/validator_functions.py b/emf/model_validator/validator_functions.py index 0f7ecf64..152b45e9 100644 --- a/emf/model_validator/validator_functions.py +++ b/emf/model_validator/validator_functions.py @@ -108,34 +108,3 @@ def check_not_retained_switches_between_nodes(original_data, open_not_retained_s original_data = triplets.rdf_parser.update_triplet_from_triplet(original_data, open_switches) return original_data, updated_switches - - -def get_not_retained_switches_between_nodes(original_data): - """ - For the loadflow open all the non-retained switches that connect different topological nodes - Currently it is seen to help around 9 to 10 Kirchhoff 1st law errors from 2 TSOs - :param original_data: original models in triplets format - :return: updated original data - """ - updated_switches = False - original_models = load_opdm_objects_to_triplets(opdm_objects=original_data) - not_retained_switches = original_models[(original_models['KEY'] == 'Switch.retained') - & (original_models['VALUE'] == "false")][['ID']] - closed_switches = original_models[(original_models['KEY'] == 'Switch.open') - & (original_models['VALUE'] == 'false')] - not_retained_closed = not_retained_switches.merge(closed_switches[['ID']], on='ID') - terminals = original_models.type_tableview('Terminal').rename_axis('Terminal').reset_index() - terminals = terminals[['Terminal', - # 'ACDCTerminal.connected', - 'Terminal.ConductingEquipment', - 'Terminal.TopologicalNode']] - not_retained_terminals = (terminals.rename(columns={'Terminal.ConductingEquipment': 'ID'}) - .merge(not_retained_closed, on='ID')) - if not_retained_terminals.empty: - return original_data, updated_switches - between_tn = ((not_retained_terminals.groupby('ID')[['Terminal.TopologicalNode']] - .apply(lambda x: check_switch_terminals(x, 'Terminal.TopologicalNode'))) - .reset_index(name='same_TN')) - between_tn = between_tn[between_tn['same_TN']] - - return between_tn From cf05c2c5ca083511c11fcd778684a43f1d5f7bde Mon Sep 17 00:00:00 2001 From: mr0321 <145195994+mr0321@users.noreply.github.com> Date: Fri, 27 Jun 2025 12:10:57 +0300 Subject: [PATCH 27/67] fix for load files --- emf/common/loadflow_tool/load_files_general.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/emf/common/loadflow_tool/load_files_general.py b/emf/common/loadflow_tool/load_files_general.py index c3179f19..a7e7932b 100644 --- a/emf/common/loadflow_tool/load_files_general.py +++ b/emf/common/loadflow_tool/load_files_general.py @@ -1,7 +1,8 @@ import os import logging -from emf.common.loadflow_tool.helper import opdm_metadata_from_filename + +from emf.common.helpers.opdm_objects import get_metadata_from_file_name OPDE_COMPONENT_KEYWORD = 'opde:Component' OPDM_PROFILE_KEYWORD = 'opdm:Profile' @@ -69,7 +70,7 @@ def get_meta_from_filename(file_name: str): if key in fixed_file_name: fixed_file_name = fixed_file_name.replace(key, BOUNDARY_FILE_TYPE_FIX[key]) # meta_data = get_metadata_from_filename(fixed_file_name) - meta_data = opdm_metadata_from_filename(fixed_file_name) + meta_data = get_metadata_from_file_name(fixed_file_name) # Revert back cases where there is a '-' in TSO's name like ENTSO-E for case in SPECIAL_TSO_NAME: if case in fixed_file_name: From 6a5d47f2b7312e41b2dd5e2e005ce4636046f875 Mon Sep 17 00:00:00 2001 From: mr0321 <145195994+mr0321@users.noreply.github.com> Date: Fri, 27 Jun 2025 12:27:44 +0300 Subject: [PATCH 28/67] various fixes to refactoring --- emf/model_merger/merge_functions.py | 83 ++++++++++++++++++++++------- emf/model_merger/model_merger.py | 15 ++++-- emf/model_merger/temporary.py | 76 +++++++++++++++++++++++++- 3 files changed, 147 insertions(+), 27 deletions(-) diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index fbf0fa9e..a98880bb 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -1,6 +1,13 @@ -import zipfile +import json import math +import os.path from io import BytesIO + +from triplets.cgmes_tools import get_metadata_from_FullModel +from triplets.rdf_parser import tableview_to_triplet, remove_triplet_from_triplet +from xml.sax.expatreader import version + +import pandas import pandas as pd import pypowsybl import logging @@ -9,15 +16,19 @@ import triplets import uuid from uuid import uuid4 + +from fontTools.merge.util import first + import config from emf.common.config_parser import parse_app_properties from emf.common.integrations import elastic -from emf.common.loadflow_tool import loadflow_settings +# from emf.common.loadflow_tool import loadflow_settings from emf.model_merger import temporary from emf.common.helpers.time import parse_datetime from emf.common.helpers.loadflow import get_model_outages, get_network_elements -from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, filename_from_opdm_metadata - +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, filename_from_opdm_metadata, \ + filename_reduced_from_opdm_metadata +from emf.model_merger.model_merger import MergedModel logger = logging.getLogger(__name__) @@ -63,9 +74,13 @@ def export_merged_model(network: pypowsybl.network, "iidm.export.cgmes.modeling-authority-set": opdm_object_meta['pmd:modelingAuthoritySet'], "iidm.export.cgmes.base-name": file_base_name, "iidm.export.cgmes.profiles": profiles, - "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids", # identity, cgmes, cgmes-fix-all-invalid-ids + # For missing instances like "SupplyStation" + # "iidm.export.cgmes.topology-kind": 'NODE_BREAKER', + # cgmes-fix-all-invalid-ids fixes non-standard uuid's. Can cause danglingReference errors + # "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids", # identity, cgmes, cgmes-fix-all-invalid-ids "iidm.export.cgmes.export-sv-injections-for-slacks": "False", - "iidm.export.cgmes.export-boundary-power-flows": "False", + # False sets all boundary flows to zero causing Kirchhoff 1st law and SvPowerFlowBranchInstances2 errors + # "iidm.export.cgmes.export-boundary-power-flows": "False", "iidm.export.cgmes.cgm_export": cgm_export_flag, } @@ -93,7 +108,7 @@ def create_merged_model_opdm_object(object_id: str, opdm_object_meta = { 'opde:Object-Type': 'CGM', 'pmd:fullModel_ID': object_id, - 'pmd:creationDate': f"{datetime.datetime.utcnow():%Y-%m-%dT%H:%M:%S.%fZ}", + 'pmd:creationDate': f"{datetime.datetime.now(datetime.UTC):%Y-%m-%dT%H:%M:%S.%fZ}", 'pmd:timeHorizon': time_horizon, 'pmd:cgmesProfile': profile, 'pmd:contentType': content_type, @@ -140,6 +155,8 @@ def update_merged_model_sv(sv_data: bytes, opdm_object_meta: dict): # Update file name at 'label' key sv_data.set_VALUE_at_KEY(key='label', value=filename_from_opdm_metadata(opdm_object_meta, file_type="xml")) + sv_data = triplets.cgmes_tools.update_FullModel_from_filename(sv_data) + # Check and fix SV id if necessary updated_sv_id_map = {} for old_id in sv_data.query("KEY == 'Type' and VALUE == 'FullModel'").ID.unique(): @@ -152,14 +169,30 @@ def update_merged_model_sv(sv_data: bytes, opdm_object_meta: dict): return sv_data -def create_updated_ssh(merged_model, original_models, models_as_triplets): +def load_ssh(input_data: pandas.DataFrame | list): + if not isinstance(input_data, pandas.DataFrame): + ssh_data = load_opdm_objects_to_triplets(input_data, "SSH") + else: + ssh_files = input_data[input_data['VALUE'] == 'SSH'][['INSTANCE_ID']].drop_duplicates() + if ssh_files.empty: + ssh_files = (input_data[(input_data['KEY'] == 'label') & + (input_data['VALUE'].str.upper().str.contains('SSH'))][['INSTANCE_ID']] + .drop_duplicates()) + ssh_data = input_data.merge(ssh_files, on='INSTANCE_ID') + ssh_data = triplets.cgmes_tools.update_FullModel_from_filename(ssh_data) + return ssh_data + + +def create_updated_ssh(models_as_triplets: pandas.DataFrame | list, + sv_data: pandas.DataFrame, + opdm_object_meta: dict + ): # TODO rewrite to use pypowsybl exported SSH ### SSH ## # Load original SSH data to created updated SSH - ssh_data = load_opdm_objects_to_triplets(opdm_objects=original_models, profile="SSH") - ssh_data = triplets.cgmes_tools.update_FullModel_from_filename(ssh_data) + ssh_data = load_ssh(models_as_triplets) # Update SSH Model.scenarioTime ssh_data.set_VALUE_at_KEY('Model.scenarioTime', opdm_object_meta['pmd:scenarioDate']) @@ -733,17 +766,22 @@ def run_post_merge_processing(input_models: list, exported_model: bytes, opdm_object_meta: dict, enable_temp_fixes: bool, + task_properties: dict = None, ): # Load original input models to triplets input_models_triplets = load_opdm_objects_to_triplets(opdm_objects=input_models) # Apply corrections to SV profile - update_merged_model_sv(sv_data=exported_model, opdm_object_meta=opdm_object_meta) + sv_data = update_merged_model_sv(sv_data=exported_model, opdm_object_meta=opdm_object_meta) # Create update SSH - sv_data, ssh_data, opdm_object_meta = create_updated_ssh() - fix_net_interchange_errors = task_properties.get('fix_net_interchange2', False) + sv_data, ssh_data, opdm_object_meta = create_updated_ssh(models_as_triplets=input_models_triplets, + sv_data=sv_data, + opdm_object_meta=opdm_object_meta) + fix_net_interchange_errors = False + if task_properties is not None: + fix_net_interchange_errors = task_properties.get('fix_net_interchange2', fix_net_interchange_errors) # Run temporary modification on exported model if enable_temp_fixes: @@ -755,32 +793,37 @@ def run_post_merge_processing(input_models: list, sv_data = temporary.check_and_fix_dependencies(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=input_models_triplets) # Run injections check and apply modification if defined in configuration + injection_threshold = float(INJECTION_THRESHOLD) + net_interchange_threshold = float(NET_INTERCHANGE_THRESHOLD) + fix_injection_errors = json.loads(str(FIX_INJECTION_ERRORS).lower()) + ssh_data= temporary.set_paired_boundary_injections_to_zero(original_models=input_models_triplets, + cgm_ssh_data=ssh_data) ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, cgm_sv_data=sv_data, original_models=input_models_triplets, injection_name='EnergySource', - threshold=INJECTION_THRESHOLD, + threshold=injection_threshold, fields_to_check={'SvPowerFlow.p': 'EnergySource.activePower'}, - fix_errors=FIX_INJECTION_ERRORS) + fix_errors=fix_injection_errors) ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, cgm_sv_data=sv_data, original_models=input_models_triplets, injection_name='ExternalNetworkInjection', fields_to_check={'SvPowerFlow.p': 'ExternalNetworkInjection.p'}, - threshold=INJECTION_THRESHOLD, - fix_errors=FIX_INJECTION_ERRORS) + threshold=injection_threshold, + fix_errors=fix_injection_errors) ssh_data = check_non_boundary_equivalent_injections(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_models=input_models_triplets, - threshold=INJECTION_THRESHOLD, - fix_errors=FIX_INJECTION_ERRORS) + threshold=injection_threshold, + fix_errors=fix_injection_errors) try: ssh_data = check_net_interchanges(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_models=input_models_triplets, fix_errors=fix_net_interchange_errors, - threshold=NET_INTERCHANGE_THRESHOLD) + threshold=net_interchange_threshold) except KeyError: logger.warning(f"No fields for net interchange correction") diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 607c995d..8e2916e0 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -1,4 +1,8 @@ import logging +import os +import sys + +import aniso8601 import pypowsybl import config import json @@ -291,6 +295,10 @@ def handle(self, task_object: dict, properties: dict, **kwargs): if json.loads(OPEN_NON_RETAINED_SWITCHES_BETWEEN_TN.lower()): pass + # Ensure boundary point EquivalentInjection are set to zero for paired tie lines + merged_model.network = merge_functions.ensure_paired_equivalent_injection_compatibility(network=merged_model.network) + + # TODO - run other LF if default fails # Run loadflow on merged model merged_model = self.run_loadflow(merged_model=merged_model) @@ -347,10 +355,6 @@ def handle(self, task_object: dict, properties: dict, **kwargs): mas=mas, version=version, ) - - # Ensure boundary point EquivalentInjection are set to zero for paired tie lines - merged_model.network = merge_functions.ensure_paired_equivalent_injection_compatibility(network=merged_model.network) - # Export merged model # TODO change here to export SSH profiles as well exported_model = merge_functions.export_merged_model(network=merged_model.network, @@ -366,6 +370,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): exported_model=exported_model, opdm_object_meta=opdm_object_meta, enable_temp_fixes=post_temp_fixes, + task_properties=task_properties ) # Package both input models and exported CGM profiles to in memory zip files @@ -397,7 +402,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): # Include original IGM files for input_model in input_models: for instance in input_model['opde:Component']: - if instance['opdm:Profile']['pmd:cgmesProfile'] in ['EQ', 'TP', 'EQBD', 'TPBD']: + if instance['opdm:Profile']['pmd:cgmesProfile'] in ['EQ', 'TP', 'EQBD', 'TPBD', 'EQ_BD', 'TP_BD']: file_object = get_opdm_component_data_bytes(opdm_component=instance) logging.info(f"Adding file: {file_object.name}") merged_model_zip.writestr(file_object.name, file_object.getvalue()) diff --git a/emf/model_merger/temporary.py b/emf/model_merger/temporary.py index aa800970..a354f581 100644 --- a/emf/model_merger/temporary.py +++ b/emf/model_merger/temporary.py @@ -1,3 +1,4 @@ +import pandas import pypowsybl import triplets import pandas as pd @@ -91,7 +92,9 @@ def check_and_fix_dependencies(cgm_sv_data, cgm_ssh_data, original_data): :param original_data: original models, will be used to get TP dependencies :return updated merged SV profile """ - some_data = load_opdm_objects_to_triplets(opdm_objects=original_data) + + # some_data = load_opdm_objects_to_triplets(opdm_objects=original_data) + some_data = get_opdm_data_from_models(model_data=original_data) tp_file_ids = some_data[(some_data['KEY'] == 'Model.profile') & (some_data['VALUE'].str.contains('Topology'))] ssh_file_ids = cgm_ssh_data[(cgm_ssh_data['KEY'] == 'Model.profile') & @@ -209,6 +212,17 @@ def take_best_match_for_sv_voltage(input_data, column_name: str = 'v', to_keep: return first_row +def get_opdm_data_from_models(model_data: list | pandas.DataFrame): + """ + Check if input is already parsed to triplets. Do it otherwise + :param model_data: input models + :return triplets + """ + if not isinstance(model_data, pandas.DataFrame): + model_data = load_opdm_objects_to_triplets(model_data) + return model_data + + def get_boundary_nodes_between_igms(model_data: list | pd.DataFrame): # TODO - EVALUATE LEGACY """ @@ -397,4 +411,62 @@ def configure_paired_boundarypoint_injections_by_nodes(data): updated_q_value["KEY"] = "EquivalentInjection.q" updated_q_value["VALUE"] = 0 - return data.update_triplet_from_triplet(pd.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) \ No newline at end of file + return data.update_triplet_from_triplet(pd.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) + + +def set_paired_boundary_injections_to_zero(original_models, cgm_ssh_data): + """Where there are paired boundary points, equivalent injections need to be modified + Set P and Q to 0 - so that no additional consumption or production is on tie line + Set voltage control off - so that no additional consumption or production is on tie line + Set terminal to connected - to be sure we have paired connected injections at boundary point + NOTE THAT THIS IS COPY FROM 'configure_paired_boundarypoint_injections' + In some models terminals are missing references to ConnectivityNodes + """ + + topological_boundary_points = original_models.query("KEY == 'TopologicalNode.boundaryPoint' and VALUE == 'true'")[["ID"]] + try: + terminals = original_models.type_tableview("Terminal").reset_index()[['ID', + 'Terminal.ConductingEquipment', + 'Terminal.ConnectivityNode', + 'Terminal.TopologicalNode']] + except KeyError: + terminals = original_models.type_tableview("Terminal").reset_index()[['ID', + 'Terminal.ConductingEquipment', + 'Terminal.TopologicalNode']] + injections = original_models.type_tableview('EquivalentInjection').reset_index()[['ID', + # 'EquivalentInjection.p', + # 'EquivalentInjection.q', + # 'EquivalentInjection.regulationStatus' + ]] + topological_boundary_points = topological_boundary_points.merge(terminals, + left_on="ID", + right_on="Terminal.TopologicalNode", + suffixes=('_TopologicalNode', '_Terminal')) + topological_injections = injections.merge(topological_boundary_points, + left_on="ID", + right_on='Terminal.ConductingEquipment', + suffixes=('_ConnectivityNode', '')) + paired_topological_injections = (topological_injections.groupby("Terminal.TopologicalNode") + .filter(lambda x: len(x) == 2)) + paired_injections = paired_topological_injections + + # Set terminal status + updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) + updated_terminal_status["KEY"] = "ACDCTerminal.connected" + updated_terminal_status["VALUE"] = "true" + + # Set Regulation off + updated_regulation_status = paired_injections[["ID"]].copy() + updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" + updated_regulation_status["VALUE"] = "false" + + # Set P to 0 + updated_p_value = paired_injections[["ID"]].copy() + updated_p_value["KEY"] = "EquivalentInjection.p" + updated_p_value["VALUE"] = 0 + + # Set Q to 0 + updated_q_value = paired_injections[["ID"]].copy() + updated_q_value["KEY"] = "EquivalentInjection.q" + updated_q_value["VALUE"] = 0 + return cgm_ssh_data.update_triplet_from_triplet(pandas.concat([updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) \ No newline at end of file From 975a9eba963b06b1cb9e15e56b831dbd802d17da Mon Sep 17 00:00:00 2001 From: mr0321 <145195994+mr0321@users.noreply.github.com> Date: Fri, 27 Jun 2025 13:44:42 +0300 Subject: [PATCH 29/67] updates dependencies --- emf/model_merger/merge_functions.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index a98880bb..827a6b42 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -17,8 +17,6 @@ import uuid from uuid import uuid4 -from fontTools.merge.util import first - import config from emf.common.config_parser import parse_app_properties from emf.common.integrations import elastic @@ -26,8 +24,7 @@ from emf.model_merger import temporary from emf.common.helpers.time import parse_datetime from emf.common.helpers.loadflow import get_model_outages, get_network_elements -from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, filename_from_opdm_metadata, \ - filename_reduced_from_opdm_metadata +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, filename_from_opdm_metadata from emf.model_merger.model_merger import MergedModel logger = logging.getLogger(__name__) From 06e91f42a81252f8db746782025cb85b03c0ebf8 Mon Sep 17 00:00:00 2001 From: mr0321 <145195994+mr0321@users.noreply.github.com> Date: Fri, 27 Jun 2025 14:19:41 +0300 Subject: [PATCH 30/67] adds export parameter for pypowsybl 1.11.1 --- emf/model_merger/merge_functions.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index fbf0fa9e..be6e29a6 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -63,9 +63,13 @@ def export_merged_model(network: pypowsybl.network, "iidm.export.cgmes.modeling-authority-set": opdm_object_meta['pmd:modelingAuthoritySet'], "iidm.export.cgmes.base-name": file_base_name, "iidm.export.cgmes.profiles": profiles, - "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids", # identity, cgmes, cgmes-fix-all-invalid-ids + # For missing instances like "SupplyStation" + "iidm.export.cgmes.topology-kind": 'NODE_BREAKER', + # cgmes-fix-all-invalid-ids fixes non-standard uuid's. Can cause danglingReference errors + # "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids", # identity, cgmes, cgmes-fix-all-invalid-ids "iidm.export.cgmes.export-sv-injections-for-slacks": "False", - "iidm.export.cgmes.export-boundary-power-flows": "False", + # False sets all boundary flows to zero causing Kirchhoff 1st law and SvPowerFlowBranchInstances2 errors + # "iidm.export.cgmes.export-boundary-power-flows": "False", "iidm.export.cgmes.cgm_export": cgm_export_flag, } From 7596e6eb393184e97c15bb83f0196f658ad9d0b0 Mon Sep 17 00:00:00 2001 From: mr0321 <145195994+mr0321@users.noreply.github.com> Date: Fri, 27 Jun 2025 15:14:38 +0300 Subject: [PATCH 31/67] removes duplicates --- emf/model_merger/merge_functions.py | 9 ++++----- emf/model_merger/temporary.py | 7 +++---- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index 827a6b42..0afb4b21 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -7,7 +7,6 @@ from triplets.rdf_parser import tableview_to_triplet, remove_triplet_from_triplet from xml.sax.expatreader import version -import pandas import pandas as pd import pypowsybl import logging @@ -166,8 +165,8 @@ def update_merged_model_sv(sv_data: bytes, opdm_object_meta: dict): return sv_data -def load_ssh(input_data: pandas.DataFrame | list): - if not isinstance(input_data, pandas.DataFrame): +def load_ssh(input_data: pd.DataFrame | list): + if not isinstance(input_data, pd.DataFrame): ssh_data = load_opdm_objects_to_triplets(input_data, "SSH") else: ssh_files = input_data[input_data['VALUE'] == 'SSH'][['INSTANCE_ID']].drop_duplicates() @@ -180,8 +179,8 @@ def load_ssh(input_data: pandas.DataFrame | list): return ssh_data -def create_updated_ssh(models_as_triplets: pandas.DataFrame | list, - sv_data: pandas.DataFrame, +def create_updated_ssh(models_as_triplets: pd.DataFrame | list, + sv_data: pd.DataFrame, opdm_object_meta: dict ): # TODO rewrite to use pypowsybl exported SSH diff --git a/emf/model_merger/temporary.py b/emf/model_merger/temporary.py index a354f581..0eb0b557 100644 --- a/emf/model_merger/temporary.py +++ b/emf/model_merger/temporary.py @@ -1,4 +1,3 @@ -import pandas import pypowsybl import triplets import pandas as pd @@ -212,13 +211,13 @@ def take_best_match_for_sv_voltage(input_data, column_name: str = 'v', to_keep: return first_row -def get_opdm_data_from_models(model_data: list | pandas.DataFrame): +def get_opdm_data_from_models(model_data: list | pd.DataFrame): """ Check if input is already parsed to triplets. Do it otherwise :param model_data: input models :return triplets """ - if not isinstance(model_data, pandas.DataFrame): + if not isinstance(model_data, pd.DataFrame): model_data = load_opdm_objects_to_triplets(model_data) return model_data @@ -469,4 +468,4 @@ def set_paired_boundary_injections_to_zero(original_models, cgm_ssh_data): updated_q_value = paired_injections[["ID"]].copy() updated_q_value["KEY"] = "EquivalentInjection.q" updated_q_value["VALUE"] = 0 - return cgm_ssh_data.update_triplet_from_triplet(pandas.concat([updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) \ No newline at end of file + return cgm_ssh_data.update_triplet_from_triplet(pd.concat([updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) \ No newline at end of file From 5b0b0e50633e68adf11ab4dae5e01c36af29fdc4 Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Fri, 27 Jun 2025 15:18:26 +0300 Subject: [PATCH 32/67] Model quality IGM loading bugfix --- config/model_quality/model_quality.properties | 2 +- emf/model_quality/model_quality.py | 47 ++++++++++--------- emf/model_quality/model_statistics.py | 15 +++++- 3 files changed, 40 insertions(+), 24 deletions(-) diff --git a/config/model_quality/model_quality.properties b/config/model_quality/model_quality.properties index e6468540..ddffd777 100644 --- a/config/model_quality/model_quality.properties +++ b/config/model_quality/model_quality.properties @@ -1,4 +1,4 @@ [MAIN] -INPUT_RMQ_QUEUE = model-quality.test +INPUT_RMQ_QUEUE = object-storage.models.quality ELK_QUALITY_INDEX = emfos-model-quality ELK_STATISTICS_INDEX = emfos-model-statistics \ No newline at end of file diff --git a/emf/model_quality/model_quality.py b/emf/model_quality/model_quality.py index 0e809db6..e6074652 100644 --- a/emf/model_quality/model_quality.py +++ b/emf/model_quality/model_quality.py @@ -4,8 +4,10 @@ import json import pypowsybl as pp from emf.common.config_parser import parse_app_properties +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets from emf.common.integrations import elastic, minio_api from emf.common.integrations.object_storage import models +from emf.common.helpers import opdm_objects from io import BytesIO from zipfile import ZipFile, ZIP_DEFLATED from triplets.rdf_parser import load_all_to_dataframe @@ -24,26 +26,24 @@ def __init__(self): def handle(self, message: bytes, properties: dict, **kwargs): - logger.info(f"Loaded {message}") - # Load OPDM metadata objects from binary to json - model_meta = json.loads(message) + model_metadata = json.loads(message) object_type = properties.headers['opde:Object-Type'] if object_type == 'CGM': - model_data = self.minio_service.download_object(model_meta.get('minio-bucket'), - model_meta.get('content_reference')) - logger.info(f"Loading merged model: {model_meta['name']}") + model_data = self.minio_service.download_object(model_metadata.get('minio-bucket'), + model_metadata.get('content_reference')) + logger.info(f"Loading merged model: {model_metadata['name']}") unzipped = process_zipped_cgm(model_data) - network = load_all_to_dataframe(unzipped) + network= load_all_to_dataframe(unzipped) # TODO test if IGM retrieving works as intended elif object_type == 'IGM': - model_meta = models.get_content(metadata=model_meta['opdm_object']) - logger.info(f"Loading model...") + latest_boundary = models.get_latest_boundary() + model_data = [models.get_content(metadata=opdm_object) for opdm_object in model_metadata] try: - data = BytesIO(model_meta["opdm:Profile"]["DATA"]) - network = load_all_to_dataframe(data) + for opdm_object in model_data: + network = load_opdm_objects_to_triplets(opdm_objects=[opdm_object, latest_boundary]) except: logger.error("Failed to load IGM") network = pd.DataFrame @@ -52,24 +52,26 @@ def handle(self, message: bytes, properties: dict, **kwargs): network = pd.DataFrame if not network.empty: - qa_report = generate_quality_report(network, object_type, model_meta) + qa_report = generate_quality_report(network, object_type, model_metadata) try: # TODO move statistics function to quality functions file or move statistics file to quality directory model_statistics = get_system_metrics(network) - except: - logger.error("Failed to get model statistics") + except Exception as e: + model_statistics = None + logger.error(f"Failed to get model statistics: {e}") else: raise TypeError("Model was not loaded correctly, either missing in MinIO or incorrect data") + # TODO align naming for opdm_objects if model_statistics: - model_statistics.update({k: v for k, v in model_meta.items() if k.startswith('@')}) + model_statistics.update({k: v for k, v in opdm_object.items() if k.startswith('@')}) model_statistics.update(properties.headers) try: response = self.elastic_service.send_to_elastic(index=ELK_STATISTICS_INDEX, json_message=model_statistics) except Exception as error: logger.error(f"Statistics report sending to Elastic failed: {error}") - logger.info(f"Statistics report sent to elastic index {ELK_STATISTICS_INDEX}") + logger.info(f"Statistics report sent to elastic index: '{ELK_STATISTICS_INDEX}'") else: raise TypeError("Statistics report generator failed, data not sent") @@ -80,18 +82,18 @@ def handle(self, message: bytes, properties: dict, **kwargs): except Exception as error: logger.error(f"Validation report sending to Elastic failed: {error}") - logger.info(f"Queality report sent to elastic index {ELK_QUALITY_INDEX}") + logger.info(f"Quality report sent to elastic index: '{ELK_QUALITY_INDEX}'") else: - raise TypeError("Quality report generator failed, data not sent") + logger.error("Error, quality report generator failed, data not sent") return message, properties -def generate_quality_report(network, object_type, model_meta): +def generate_quality_report(network, object_type, model_metadata): report = {} - if object_type == "CGM" and model_meta['merge_type'] == 'BA': + if object_type == "CGM" and model_metadata['merge_type'] == 'BA': # Check Kruonis generators generators = network.type_tableview('SynchronousMachine').rename_axis('Terminal').reset_index() @@ -171,7 +173,10 @@ def generate_quality_report(network, object_type, model_meta): # TODO define IGM quality rules elif object_type == "IGM": - report = report + report = model_metadata[0] + model_metadata[0].pop('opde:Component') + model_metadata[0].pop('opde:Dependencies') + report.update({"quality": "No Status"}) return report diff --git a/emf/model_quality/model_statistics.py b/emf/model_quality/model_statistics.py index 1daa072b..7c7ab824 100644 --- a/emf/model_quality/model_statistics.py +++ b/emf/model_quality/model_statistics.py @@ -87,8 +87,19 @@ def parse_type_and_attribute(type_and_attr_string, default_attr="ID"): def get_tieflow_data(data): logger.info("Getting Tieflow Data") - tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal->ConnectivityNode") - tieflow_data["BoundaryPoint.isDirectCurrent"] = tieflow_data["IdentifiedObject.description_ConnectivityNode"].str.startswith("HVDC") + try: + tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal->ConnectivityNode") + except: + tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal-ConnectivityNode") + + try: + tieflow_data["BoundaryPoint.isDirectCurrent"] = tieflow_data["IdentifiedObject.description"].str.startswith("HVDC") + except: + try: + tieflow_data["BoundaryPoint.isDirectCurrent"] = tieflow_data["IdentifiedObject.description_ConnectivityNode"].str.startswith("HVDC") + except: + tieflow_data["BoundaryPoint.isDirectCurrent"] = False + # TODO - for CGMES3/CIM17 get also the Boundary objects and use correct field to identify HVDC # Add Injections From a1c320462150bffdfd0e219470cc059ba247ee27 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 30 Jun 2025 10:01:24 +0300 Subject: [PATCH 33/67] logging: added log message to report if LF diverged after island ACNP alignment --- emf/model_merger/scaler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/emf/model_merger/scaler.py b/emf/model_merger/scaler.py index 2b8199d0..945793a0 100644 --- a/emf/model_merger/scaler.py +++ b/emf/model_merger/scaler.py @@ -252,6 +252,7 @@ def scale_balance(model: object, # TODO need to consider how to evaluate it in case of multiple islands. For example if one of the island diverges but not the main if not validate_loadflow_status(results=pf_results): model.scaled = False + logger.warning(f"Terminating network scaling due to divergence in main island after island ACNP alignment") return model # Validate total network AC net position alignment From 0614a36a46a7d218b7e0381a2a3214aa2a6136d2 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 30 Jun 2025 10:20:08 +0300 Subject: [PATCH 34/67] remove temporary metrics collection as this has been moved to statistics module --- emf/model_validator/model_validator.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index 1e26deee..23cd4d14 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -246,17 +246,6 @@ def handle(self, message: bytes, properties: dict, **kwargs): except Exception as error: logger.error(f"Models validator failed with exception: {error}", exc_info=True) - # TODO temporary monitored metrics - if opdm_object['pmd:TSO'] == "LITGRID": - try: - dl = network.get_dangling_lines() - ltpl_border_flow = dl.query("pairing_key in ['XEL_AL11', 'XEL_AL12']").p.sum() - gens = network.get_generators() - khae_gen = gens[gens['name'].str.contains('KHAE_G')].p.sum() - report["monitor"] = {"poland_border_flow": ltpl_border_flow, "khae_generation": khae_gen} - except Exception as error: - logger.error(f"Monitored metrics collection failed: {error}", exc_info=True) - # Define model validity valid = all(report['validations'].values()) From e1d9dda5dd333fc8fff3f824b4496cb214cdde94 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 30 Jun 2025 13:27:54 +0300 Subject: [PATCH 35/67] remove pre validator regarding kirchoff law --- emf/model_validator/model_validator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index 23cd4d14..fa73c412 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -45,6 +45,7 @@ def validate_non_retained_switches(self, open_non_retained_switches: bool = Fals self.report['pre_validations']['non_retained_switches'] = non_retained_switched_valid def validate_kirchhoff_first_law(self): + # TODO - currently this is not used in pre-validation, but it might be useful to have it here violated_nodes = validator_functions.get_nodes_against_kirchhoff_first_law(original_models=self.network) kirchhoff_first_law_valid = True if violated_nodes.empty else False self.report['pre_validations']['kirchhoff_first_law'] = kirchhoff_first_law_valid @@ -56,8 +57,6 @@ def run_validation(self): open_non_retained_switches = False if json.loads(CHECK_NON_RETAINED_SWITCHES.lower()): self.validate_non_retained_switches(open_non_retained_switches=open_non_retained_switches) - if json.loads(CHECK_KIRCHHOFF_FIRST_LAW.lower()): - self.validate_kirchhoff_first_law() class PostLFValidator: From 6030af1d20c99f6f4234c9c0c02d03dc1ee8bec5 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 30 Jun 2025 13:28:13 +0300 Subject: [PATCH 36/67] reafactor: moved function to helpers.utils --- emf/common/helpers/utils.py | 14 ++++++++++++++ emf/model_merger/model_merger.py | 20 +------------------- 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/emf/common/helpers/utils.py b/emf/common/helpers/utils.py index 64d82880..dca06a24 100644 --- a/emf/common/helpers/utils.py +++ b/emf/common/helpers/utils.py @@ -63,6 +63,20 @@ def flatten_dict(nested_dict: dict, parent_key: str = '', separator: str = '.'): return dict(items) +def convert_dict_str_to_bool(data_dict: dict): + for key, value in data_dict.items(): + if isinstance(value, str): + if value in ['True', 'true', 'TRUE']: + data_dict[key] = True + elif value in ['False', 'false', 'FALSE']: + data_dict[key] = False + elif isinstance(value, dict): + # Recursively converter nested dictionaries + data_dict[key] = convert_dict_str_to_bool(value) + + return data_dict + + def filter_and_flatten_dict(nested_dict: dict, keys: list): """ Creates a new flat dictionary from specified keys. diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 8e2916e0..1007e320 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -1,8 +1,4 @@ import logging -import os -import sys - -import aniso8601 import pypowsybl import config import json @@ -19,7 +15,7 @@ from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download from emf.common.integrations.object_storage.schedules import query_acnp_schedules, query_hvdc_schedules from emf.common.loadflow_tool import loadflow_settings -from emf.common.helpers.utils import attr_to_dict +from emf.common.helpers.utils import attr_to_dict, convert_dict_str_to_bool from emf.common.helpers.cgmes import export_to_cgmes_zip from emf.common.helpers.opdm_objects import get_opdm_component_data_bytes from emf.common.helpers.loadflow import load_network_model @@ -49,20 +45,6 @@ def log_opdm_response(response): logger.debug(etree.tostring(response, pretty_print=True).decode()) -def convert_dict_str_to_bool(data_dict: dict): - for key, value in data_dict.items(): - if isinstance(value, str): - if value in ['True', 'true', 'TRUE']: - data_dict[key] = True - elif value in ['False', 'false', 'FALSE']: - data_dict[key] = False - elif isinstance(value, dict): - # Recursively converter nested dictionaries - data_dict[key] = convert_dict_str_to_bool(value) - - return data_dict - - @dataclass class MergedModel: network: pypowsybl.network = None From a70d70a00983a1224443c7e1c7a776e7f185b7b3 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 30 Jun 2025 14:22:36 +0300 Subject: [PATCH 37/67] feature: tie lines consistency of connectivity function --- emf/model_merger/merge_functions.py | 22 +++++++++++++++++++++- emf/model_merger/model_merger.py | 2 ++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index 0afb4b21..0b6a072b 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -287,7 +287,6 @@ def ensure_paired_equivalent_injection_compatibility(network: pypowsybl.network) LEVEL7 rule PairedEICompatibility Set P and Q to 0 - so that no additional consumption or production is on tie line - TODO consider handling if same tie line EQINJ has different ACDCTerminal connected status """ logger.info("Configuring paired boundary points equivalent injections: p0/q0 = 0.0") dangling_lines = network.get_dangling_lines(all_attributes=True) @@ -299,6 +298,27 @@ def ensure_paired_equivalent_injection_compatibility(network: pypowsybl.network) return network +def ensure_paired_boundary_line_connectivity(network: pypowsybl.network): + logger.info("Aligning paired boundary lines connection status") + dangling_lines = network.get_dangling_lines(all_attributes=True) + paired_dangling_lines = dangling_lines[dangling_lines['paired'] == True] + + # Identify dangling line pairs where the 'connected' status is inconsistent within each pairing_key group + mask = paired_dangling_lines.groupby('pairing_key')['connected'].transform(lambda s: s.nunique() > 1) + mismatched_dangling_lines = paired_dangling_lines[mask] + logger.info(f"Boundary lines with non-matching connection status: {mismatched_dangling_lines['pairing_key'].unique().tolist()}") + + # Set all mismatched lines to disconnected (False) + _connected = pd.Series(data=False, index=mismatched_dangling_lines.index) + network.update_dangling_lines(id=mismatched_dangling_lines.index, connected=_connected) + + # Log each change + for i, row in mismatched_dangling_lines.iterrows(): + logger.info(f"Changed status of dangling line {row['name']}: {row['connected']} -> False") + + return network + + def generate_merge_report(merged_model: object, task: dict): """ Creates JSON type report of pypowsybl loadflow results diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 1007e320..50acddb4 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -280,6 +280,8 @@ def handle(self, task_object: dict, properties: dict, **kwargs): # Ensure boundary point EquivalentInjection are set to zero for paired tie lines merged_model.network = merge_functions.ensure_paired_equivalent_injection_compatibility(network=merged_model.network) + # Ensure boundary line connectivity consistency for paired boundary lines + merged_model.network = merge_functions.ensure_paired_boundary_line_connectivity(network=merged_model.network) # TODO - run other LF if default fails # Run loadflow on merged model From c48f8257a05ebd6177c2114360379fd328fce825 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 30 Jun 2025 15:37:45 +0300 Subject: [PATCH 38/67] refactor: not used temporary functions moved to legacy.py file logging: refined some log messages --- emf/common/helpers/tasks.py | 2 +- emf/model_merger/legacy.py | 130 +++++++++++++++++++++++++ emf/model_merger/merge_functions.py | 14 ++- emf/model_merger/model_merger.py | 3 +- emf/model_merger/temporary.py | 145 ++-------------------------- 5 files changed, 150 insertions(+), 144 deletions(-) create mode 100644 emf/model_merger/legacy.py diff --git a/emf/common/helpers/tasks.py b/emf/common/helpers/tasks.py index f4e44c8c..0044de32 100644 --- a/emf/common/helpers/tasks.py +++ b/emf/common/helpers/tasks.py @@ -14,7 +14,7 @@ def update_task_status(task: dict, status_text: str, publish: bool = True): Will update task_status Will append new status to task_status_trace""" - logger.info(f"Updating Task status to: {status_text}") + logger.info(f"Updating task status to: {status_text}") utc_now = datetime.utcnow().isoformat() diff --git a/emf/model_merger/legacy.py b/emf/model_merger/legacy.py new file mode 100644 index 00000000..8e8a9797 --- /dev/null +++ b/emf/model_merger/legacy.py @@ -0,0 +1,130 @@ +import pandas as pd +import triplets + + +def configure_paired_boundarypoint_injections(data): + # TODO [LEGACY] + """Where there are paired boundary points, equivalent injections need to be modified + Set P and Q to 0 - so that no additional consumption or production is on tie line + Set voltage control off - so that no additional consumption or production is on tie line + Set terminal to connected - to be sure we have paired connected injections at boundary point + """ + boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] + boundary_points = boundary_points.merge(data.type_tableview("Terminal").reset_index(), + left_on="ID", + right_on="Terminal.ConnectivityNode", + suffixes=('_ConnectivityNode', '_Terminal')) + injections = data.type_tableview('EquivalentInjection').reset_index().merge(boundary_points, + left_on="ID", + right_on='Terminal.ConductingEquipment', + suffixes=('_ConnectivityNode', '')) + + # Get paired injections at boundary points + paired_injections = injections.groupby("Terminal.ConnectivityNode").filter(lambda x: len(x) == 2) + + # Set terminal status + updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) + updated_terminal_status["KEY"] = "ACDCTerminal.connected" + updated_terminal_status["VALUE"] = "true" + + # Set Regulation off + updated_regulation_status = paired_injections[["ID"]].copy() + updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" + updated_regulation_status["VALUE"] = "false" + + # Set P to 0 + updated_p_value = paired_injections[["ID"]].copy() + updated_p_value["KEY"] = "EquivalentInjection.p" + updated_p_value["VALUE"] = 0 + + # Set Q to 0 + updated_q_value = paired_injections[["ID"]].copy() + updated_q_value["KEY"] = "EquivalentInjection.q" + updated_q_value["VALUE"] = 0 + + return data.update_triplet_from_triplet(pd.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) + + +def configure_paired_boundarypoint_injections_by_nodes(data): + # TODO [LEGACY] + """Where there are paired boundary points, equivalent injections need to be modified + Set P and Q to 0 - so that no additional consumption or production is on tie line + Set voltage control off - so that no additional consumption or production is on tie line + Set terminal to connected - to be sure we have paired connected injections at boundary point + + TODO NOTE THAT THIS IS COPY FROM 'configure_paired_boundarypoint_injections' + In some models terminals are missing references to ConnectivityNodes + """ + connectivity_boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] + topological_boundary_points = data.query("KEY == 'TopologicalNode.boundaryPoint' and VALUE == 'true'")[["ID"]] + try: + terminals = data.type_tableview("Terminal").reset_index()[['ID', + 'Terminal.ConductingEquipment', + 'Terminal.ConnectivityNode', + 'Terminal.TopologicalNode']] + except KeyError: + terminals = data.type_tableview("Terminal").reset_index()[['ID', + 'Terminal.ConductingEquipment', + 'Terminal.TopologicalNode']] + injections = data.type_tableview('EquivalentInjection').reset_index()[['ID', + # 'EquivalentInjection.p', + # 'EquivalentInjection.q', + # 'EquivalentInjection.regulationStatus' + ]] + topological_boundary_points = topological_boundary_points.merge(terminals, + left_on="ID", + right_on="Terminal.TopologicalNode", + suffixes=('_TopologicalNode', '_Terminal')) + topological_injections = injections.merge(topological_boundary_points, + left_on="ID", + right_on='Terminal.ConductingEquipment', + suffixes=('_ConnectivityNode', '')) + paired_topological_injections = (topological_injections.groupby("Terminal.TopologicalNode") + .filter(lambda x: len(x) == 2)) + paired_injections = paired_topological_injections + if 'Terminal.ConnectivityNode' in terminals: + connectivity_boundary_points = connectivity_boundary_points.merge(terminals, + left_on="ID", + right_on="Terminal.ConnectivityNode", + suffixes=('_ConnectivityNode', '_Terminal')) + connectivity_injections = injections.merge(connectivity_boundary_points, + left_on="ID", + right_on='Terminal.ConductingEquipment', + suffixes=('_TopologicalNode', '')) + + paired_connectivity_injections = (connectivity_injections.groupby("Terminal.ConnectivityNode") + .filter(lambda x: len(x) == 2)) + merged_injections = paired_connectivity_injections.merge(paired_topological_injections, + on='ID', + how='outer', + indicator=True, + suffixes=('_CN', '_TN')) + only_connectivity_injections = merged_injections[merged_injections['_merge'] == 'left_only'] + only_topological_injections = merged_injections[merged_injections['_merge'] == 'right_only'] + if len(only_connectivity_injections.index) != 0 or len(only_topological_injections.index) == 0: + paired_injections = paired_connectivity_injections + else: + logger.warning(f"Mismatch of finding paired injections from topological nodes and connectivity nodes") + else: + logger.warning(f"Terminals do not contain Connectivity nodes") + # Set terminal status + updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) + updated_terminal_status["KEY"] = "ACDCTerminal.connected" + updated_terminal_status["VALUE"] = "true" + + # Set Regulation off + updated_regulation_status = paired_injections[["ID"]].copy() + updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" + updated_regulation_status["VALUE"] = "false" + + # Set P to 0 + updated_p_value = paired_injections[["ID"]].copy() + updated_p_value["KEY"] = "EquivalentInjection.p" + updated_p_value["VALUE"] = 0 + + # Set Q to 0 + updated_q_value = paired_injections[["ID"]].copy() + updated_q_value["KEY"] = "EquivalentInjection.q" + updated_q_value["VALUE"] = 0 + + return data.update_triplet_from_triplet(pd.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) \ No newline at end of file diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index 0b6a072b..16ff7ce1 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -159,7 +159,7 @@ def update_merged_model_sv(sv_data: bytes, opdm_object_meta: dict): if not is_valid_uuid(old_id): new_id = str(uuid4()) updated_sv_id_map[old_id] = new_id - logger.warning(f"SV id {old_id} is not valid, assigning: {new_id}") + logger.warning(f"SV profile id {old_id} is not valid, assigning: {new_id}") sv_data = sv_data.replace(updated_sv_id_map) return sv_data @@ -291,6 +291,11 @@ def ensure_paired_equivalent_injection_compatibility(network: pypowsybl.network) logger.info("Configuring paired boundary points equivalent injections: p0/q0 = 0.0") dangling_lines = network.get_dangling_lines(all_attributes=True) paired_dangling_lines = dangling_lines[dangling_lines['paired'] == True] + if paired_dangling_lines.empty: + logger.warning(f"No paired dangling lines found in network model") + return network + + # Set p0/q0 to 0 for all paired dangling lines _updated_p0 = pd.Series(0, index=paired_dangling_lines.index) _updated_q0 = pd.Series(0, index=paired_dangling_lines.index) network.update_dangling_lines(id=paired_dangling_lines.index, p0=_updated_p0, q0=_updated_q0) @@ -302,6 +307,9 @@ def ensure_paired_boundary_line_connectivity(network: pypowsybl.network): logger.info("Aligning paired boundary lines connection status") dangling_lines = network.get_dangling_lines(all_attributes=True) paired_dangling_lines = dangling_lines[dangling_lines['paired'] == True] + if paired_dangling_lines.empty: + logger.warning(f"No paired dangling lines found in network model") + return network # Identify dangling line pairs where the 'connected' status is inconsistent within each pairing_key group mask = paired_dangling_lines.groupby('pairing_key')['connected'].transform(lambda s: s.nunique() > 1) @@ -812,8 +820,8 @@ def run_post_merge_processing(input_models: list, injection_threshold = float(INJECTION_THRESHOLD) net_interchange_threshold = float(NET_INTERCHANGE_THRESHOLD) fix_injection_errors = json.loads(str(FIX_INJECTION_ERRORS).lower()) - ssh_data= temporary.set_paired_boundary_injections_to_zero(original_models=input_models_triplets, - cgm_ssh_data=ssh_data) + ssh_data = temporary.set_paired_boundary_injections_to_zero(original_models=input_models_triplets, + cgm_ssh_data=ssh_data) ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, cgm_sv_data=sv_data, original_models=input_models_triplets, diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 50acddb4..2780a6c2 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -478,7 +478,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): "job_period_start": "2024-05-24T22:00:00+00:00", "job_period_end": "2024-05-25T06:00:00+00:00", "task_properties": { - "timestamp_utc": "2025-06-16T12:30:00+00:00", + "timestamp_utc": "2025-06-30T12:30:00+00:00", "merge_type": "BA", "merging_entity": "BALTICRCC", "included": ["LITGRID", "AST", "ELERING", "PSE"], @@ -487,7 +487,6 @@ def handle(self, task_object: dict, properties: dict, **kwargs): "time_horizon": "1D", "version": "001", "mas": "http://www.baltic-rsc.eu/OperationalPlanning", - "pre_temp_fixes": "True", "post_temp_fixes": "True", "fix_net_interchange2": "True", "replacement": "True", diff --git a/emf/model_merger/temporary.py b/emf/model_merger/temporary.py index 0eb0b557..7aac59bb 100644 --- a/emf/model_merger/temporary.py +++ b/emf/model_merger/temporary.py @@ -106,8 +106,7 @@ def check_and_fix_dependencies(cgm_sv_data, cgm_ssh_data, original_data): cgm_sv_data = triplets.rdf_parser.remove_triplet_from_triplet(cgm_sv_data, existing_dependencies) full_model_id = cgm_sv_data[(cgm_sv_data['KEY'] == 'Type') & (cgm_sv_data['VALUE'] == 'FullModel')] dependencies_to_update = dependency_difference.query('_merge != "left_only"') - logger.info(f"Mismatch of dependencies. Inserting {len(dependencies_to_update.index)} " - f"dependencies to SV profile") + logger.warning(f"Mismatch of dependencies. Inserting {len(dependencies_to_update.index)} dependencies to SV profile") new_dependencies = dependencies_to_update[['VALUE']].copy().reset_index(drop=True) new_dependencies.loc[:, 'KEY'] = 'Model.DependentOn' new_dependencies.loc[:, 'ID'] = full_model_id['ID'].iloc[0] @@ -284,154 +283,24 @@ def remove_duplicate_sv_voltages(cgm_sv_data, original_data): return cgm_sv_data -################################################### LEGACY FUNCTIONS ################################################### -def configure_paired_boundarypoint_injections(data): - # TODO [LEGACY] - """Where there are paired boundary points, equivalent injections need to be modified - Set P and Q to 0 - so that no additional consumption or production is on tie line - Set voltage control off - so that no additional consumption or production is on tie line - Set terminal to connected - to be sure we have paired connected injections at boundary point - """ - boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] - boundary_points = boundary_points.merge(data.type_tableview("Terminal").reset_index(), - left_on="ID", - right_on="Terminal.ConnectivityNode", - suffixes=('_ConnectivityNode', '_Terminal')) - injections = data.type_tableview('EquivalentInjection').reset_index().merge(boundary_points, - left_on="ID", - right_on='Terminal.ConductingEquipment', - suffixes=('_ConnectivityNode', '')) - - # Get paired injections at boundary points - paired_injections = injections.groupby("Terminal.ConnectivityNode").filter(lambda x: len(x) == 2) - - # Set terminal status - updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) - updated_terminal_status["KEY"] = "ACDCTerminal.connected" - updated_terminal_status["VALUE"] = "true" - - # Set Regulation off - updated_regulation_status = paired_injections[["ID"]].copy() - updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" - updated_regulation_status["VALUE"] = "false" - - # Set P to 0 - updated_p_value = paired_injections[["ID"]].copy() - updated_p_value["KEY"] = "EquivalentInjection.p" - updated_p_value["VALUE"] = 0 - - # Set Q to 0 - updated_q_value = paired_injections[["ID"]].copy() - updated_q_value["KEY"] = "EquivalentInjection.q" - updated_q_value["VALUE"] = 0 - - return data.update_triplet_from_triplet(pd.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) - - -def configure_paired_boundarypoint_injections_by_nodes(data): - # TODO [LEGACY] - """Where there are paired boundary points, equivalent injections need to be modified - Set P and Q to 0 - so that no additional consumption or production is on tie line - Set voltage control off - so that no additional consumption or production is on tie line - Set terminal to connected - to be sure we have paired connected injections at boundary point - - TODO NOTE THAT THIS IS COPY FROM 'configure_paired_boundarypoint_injections' - In some models terminals are missing references to ConnectivityNodes - """ - connectivity_boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] - topological_boundary_points = data.query("KEY == 'TopologicalNode.boundaryPoint' and VALUE == 'true'")[["ID"]] - try: - terminals = data.type_tableview("Terminal").reset_index()[['ID', - 'Terminal.ConductingEquipment', - 'Terminal.ConnectivityNode', - 'Terminal.TopologicalNode']] - except KeyError: - terminals = data.type_tableview("Terminal").reset_index()[['ID', - 'Terminal.ConductingEquipment', - 'Terminal.TopologicalNode']] - injections = data.type_tableview('EquivalentInjection').reset_index()[['ID', - # 'EquivalentInjection.p', - # 'EquivalentInjection.q', - # 'EquivalentInjection.regulationStatus' - ]] - topological_boundary_points = topological_boundary_points.merge(terminals, - left_on="ID", - right_on="Terminal.TopologicalNode", - suffixes=('_TopologicalNode', '_Terminal')) - topological_injections = injections.merge(topological_boundary_points, - left_on="ID", - right_on='Terminal.ConductingEquipment', - suffixes=('_ConnectivityNode', '')) - paired_topological_injections = (topological_injections.groupby("Terminal.TopologicalNode") - .filter(lambda x: len(x) == 2)) - paired_injections = paired_topological_injections - if 'Terminal.ConnectivityNode' in terminals: - connectivity_boundary_points = connectivity_boundary_points.merge(terminals, - left_on="ID", - right_on="Terminal.ConnectivityNode", - suffixes=('_ConnectivityNode', '_Terminal')) - connectivity_injections = injections.merge(connectivity_boundary_points, - left_on="ID", - right_on='Terminal.ConductingEquipment', - suffixes=('_TopologicalNode', '')) - - paired_connectivity_injections = (connectivity_injections.groupby("Terminal.ConnectivityNode") - .filter(lambda x: len(x) == 2)) - merged_injections = paired_connectivity_injections.merge(paired_topological_injections, - on='ID', - how='outer', - indicator=True, - suffixes=('_CN', '_TN')) - only_connectivity_injections = merged_injections[merged_injections['_merge'] == 'left_only'] - only_topological_injections = merged_injections[merged_injections['_merge'] == 'right_only'] - if len(only_connectivity_injections.index) != 0 or len(only_topological_injections.index) == 0: - paired_injections = paired_connectivity_injections - else: - logger.warning(f"Mismatch of finding paired injections from topological nodes and connectivity nodes") - else: - logger.warning(f"Terminals do not contain Connectivity nodes") - # Set terminal status - updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) - updated_terminal_status["KEY"] = "ACDCTerminal.connected" - updated_terminal_status["VALUE"] = "true" - - # Set Regulation off - updated_regulation_status = paired_injections[["ID"]].copy() - updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" - updated_regulation_status["VALUE"] = "false" - - # Set P to 0 - updated_p_value = paired_injections[["ID"]].copy() - updated_p_value["KEY"] = "EquivalentInjection.p" - updated_p_value["VALUE"] = 0 - - # Set Q to 0 - updated_q_value = paired_injections[["ID"]].copy() - updated_q_value["KEY"] = "EquivalentInjection.q" - updated_q_value["VALUE"] = 0 - - return data.update_triplet_from_triplet(pd.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) - - def set_paired_boundary_injections_to_zero(original_models, cgm_ssh_data): """Where there are paired boundary points, equivalent injections need to be modified Set P and Q to 0 - so that no additional consumption or production is on tie line Set voltage control off - so that no additional consumption or production is on tie line Set terminal to connected - to be sure we have paired connected injections at boundary point - NOTE THAT THIS IS COPY FROM 'configure_paired_boundarypoint_injections' In some models terminals are missing references to ConnectivityNodes """ topological_boundary_points = original_models.query("KEY == 'TopologicalNode.boundaryPoint' and VALUE == 'true'")[["ID"]] try: terminals = original_models.type_tableview("Terminal").reset_index()[['ID', - 'Terminal.ConductingEquipment', - 'Terminal.ConnectivityNode', - 'Terminal.TopologicalNode']] + 'Terminal.ConductingEquipment', + 'Terminal.ConnectivityNode', + 'Terminal.TopologicalNode']] except KeyError: terminals = original_models.type_tableview("Terminal").reset_index()[['ID', - 'Terminal.ConductingEquipment', - 'Terminal.TopologicalNode']] + 'Terminal.ConductingEquipment', + 'Terminal.TopologicalNode']] injections = original_models.type_tableview('EquivalentInjection').reset_index()[['ID', # 'EquivalentInjection.p', # 'EquivalentInjection.q', @@ -468,4 +337,4 @@ def set_paired_boundary_injections_to_zero(original_models, cgm_ssh_data): updated_q_value = paired_injections[["ID"]].copy() updated_q_value["KEY"] = "EquivalentInjection.q" updated_q_value["VALUE"] = 0 - return cgm_ssh_data.update_triplet_from_triplet(pd.concat([updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) \ No newline at end of file + return cgm_ssh_data.update_triplet_from_triplet(pd.concat([updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) From 6930b74fa27c5ffc0adb9d7547aaed14d5cfd115 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 30 Jun 2025 16:36:35 +0300 Subject: [PATCH 39/67] refactor: model validator separated non retained switchis to validation and modification step --- config/cgm_worker/merger.properties | 1 - emf/model_merger/merge_functions.py | 9 ++++-- emf/model_merger/model_merger.py | 4 --- emf/model_merger/temporary.py | 3 +- emf/model_validator/model_validator.py | 37 ++++++++++------------ emf/model_validator/validator_functions.py | 14 ++++---- 6 files changed, 30 insertions(+), 38 deletions(-) diff --git a/config/cgm_worker/merger.properties b/config/cgm_worker/merger.properties index 87115bf7..36dda2ea 100644 --- a/config/cgm_worker/merger.properties +++ b/config/cgm_worker/merger.properties @@ -8,5 +8,4 @@ OUTPUT_MINIO_FOLDER = EMFOS/RMM MERGE_REPORT_ELK_INDEX = emfos-merge-reports OPDE_MODELS_ELK_INDEX = emfos-opde-models MERGE_LOAD_FLOW_SETTINGS = CGM_RELAXED_3_CUSTOM -OPEN_NON_RETAINED_SWITCHES_BETWEEN_TN = True REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION = True \ No newline at end of file diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index 16ff7ce1..0f37845b 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -807,7 +807,8 @@ def run_post_merge_processing(input_models: list, if task_properties is not None: fix_net_interchange_errors = task_properties.get('fix_net_interchange2', fix_net_interchange_errors) - # Run temporary modification on exported model + # Run temporary modifications on exported model + # Temporary fixes are applied to SV and SSH profiles if enable_temp_fixes: # TODO need to revise constantly sv_data = temporary.remove_equivalent_shunt_section(sv_data, input_models_triplets) @@ -815,13 +816,15 @@ def run_post_merge_processing(input_models: list, sv_data = temporary.remove_small_islands(sv_data, int(SMALL_ISLAND_SIZE)) sv_data = temporary.remove_duplicate_sv_voltages(cgm_sv_data=sv_data, original_data=input_models_triplets) sv_data = temporary.check_and_fix_dependencies(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=input_models_triplets) + # TODO following SSH profile fix should be removed once pypowsybl SSH export will be used + ssh_data = temporary.set_paired_boundary_injections_to_zero(original_models=input_models_triplets, + cgm_ssh_data=ssh_data) # Run injections check and apply modification if defined in configuration injection_threshold = float(INJECTION_THRESHOLD) net_interchange_threshold = float(NET_INTERCHANGE_THRESHOLD) fix_injection_errors = json.loads(str(FIX_INJECTION_ERRORS).lower()) - ssh_data = temporary.set_paired_boundary_injections_to_zero(original_models=input_models_triplets, - cgm_ssh_data=ssh_data) + ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, cgm_sv_data=sv_data, original_models=input_models_triplets, diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 2780a6c2..5c80a47c 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -273,10 +273,6 @@ def handle(self, task_object: dict, properties: dict, **kwargs): if json.loads(REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION.lower()): merged_model.network = handle_igm_ssh_vs_cgm_ssh_error(network_pre_instance=merged_model.network) - # TODO - placeholder for open TN switcher or it can be after scaling before export - if json.loads(OPEN_NON_RETAINED_SWITCHES_BETWEEN_TN.lower()): - pass - # Ensure boundary point EquivalentInjection are set to zero for paired tie lines merged_model.network = merge_functions.ensure_paired_equivalent_injection_compatibility(network=merged_model.network) diff --git a/emf/model_merger/temporary.py b/emf/model_merger/temporary.py index 7aac59bb..25371a6f 100644 --- a/emf/model_merger/temporary.py +++ b/emf/model_merger/temporary.py @@ -74,8 +74,7 @@ def open_switches_in_network(network_pre_instance: pypowsybl.network.Network, sw logger.info(f"Opening {len(switches_dataframe.index)} switches") switches = network_pre_instance.get_switches(all_attributes=True).reset_index() switches = switches.merge(switches_dataframe[['ID']].rename(columns={'ID': 'id'}), on='id') - non_retained_closed = switches.merge(switches_dataframe.rename(columns={'ID': 'id'}), - on='id')[['id', 'open']] + non_retained_closed = switches.merge(switches_dataframe.rename(columns={'ID': 'id'}), on='id')[['id', 'open']] non_retained_closed['open'] = True network_pre_instance.update_switches(non_retained_closed.set_index('id')) return network_pre_instance diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index fa73c412..ecc6bdfe 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -29,18 +29,10 @@ def __init__(self, network: pd.DataFrame): self.network = network self.report = {'pre_validations': {}} - def validate_non_retained_switches(self, open_non_retained_switches: bool = False): - # TODO Use carefully as this might change switch statuses in the original model - # TODO Currently disabled returning modified data + def validate_non_retained_switches(self): non_retained_switched_valid = True - if open_non_retained_switches: - self.network, non_retained_switches = ( - validator_functions - .check_not_retained_switches_between_nodes(original_data=self.network, - open_not_retained_switches=open_non_retained_switches)) - else: - non_retained_switches = validator_functions.check_not_retained_switches_between_nodes(self.network)[1] - if non_retained_switches: + violated_switches = validator_functions.check_not_retained_switches_between_nodes(original_data=self.network)[1] + if violated_switches: non_retained_switched_valid = False self.report['pre_validations']['non_retained_switches'] = non_retained_switched_valid @@ -51,12 +43,8 @@ def validate_kirchhoff_first_law(self): self.report['pre_validations']['kirchhoff_first_law'] = kirchhoff_first_law_valid def run_validation(self): - try: - open_non_retained_switches = json.loads(OPEN_NON_RETAINED_SWITCHES.lower()) - except (json.decoder.JSONDecodeError, AttributeError): - open_non_retained_switches = False if json.loads(CHECK_NON_RETAINED_SWITCHES.lower()): - self.validate_non_retained_switches(open_non_retained_switches=open_non_retained_switches) + self.validate_non_retained_switches() class PostLFValidator: @@ -112,9 +100,7 @@ def validate_network_elements(self): def validate_kirchhoff_first_law(self): """Validates possible Kirchhoff first law errors after loadflow""" - violated_nodes = pd.DataFrame() - - # Export sv profile and check it for Kirchhoff 1st law + # Export SV profile and check it for Kirchhoff 1st law export_parameters = {"iidm.export.cgmes.profiles": 'SV', "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids"} bytes_object = self.network.save_to_binary_buffer(format="CGMES", parameters=export_parameters) @@ -163,10 +149,21 @@ def sanitize_file_name(self): add=False) self.report['modification']['sanitize_file_name'] = True + def open_non_retained_switches(self): + self.report['modification']['open_non_retained_switches'] = False + # Open non-retained switches + original_data, violated_switches = validator_functions.check_not_retained_switches_between_nodes( + original_data=self.network, open_not_retained_switches=True) + if violated_switches: + self.network = original_data + self.report['modification']['open_non_retained_switches'] = True + @performance_counter(units='seconds') def run_pre_process_modifications(self): self.update_header_from_file_name() self.sanitize_file_name() + if json.loads(OPEN_NON_RETAINED_SWITCHES.lower()): + self.open_non_retained_switches() return self.network @@ -212,7 +209,7 @@ def handle(self, message: bytes, properties: dict, **kwargs): post_lf_validation = PostLFValidator(network=network, network_triplets=network_triplets) post_lf_validation.run_validation() - # Clean opdm object from DATA as this is already converted to other formats + # Clean DATA from OPDM object as this is already converted to other formats opdm_object = clean_data_from_opdm_objects(opdm_objects=[opdm_object])[0] # Apply pre-processing modification to models and store in Minio diff --git a/emf/model_validator/validator_functions.py b/emf/model_validator/validator_functions.py index 152b45e9..bb598b3b 100644 --- a/emf/model_validator/validator_functions.py +++ b/emf/model_validator/validator_functions.py @@ -78,7 +78,7 @@ def check_not_retained_switches_between_nodes(original_data, open_not_retained_s :param open_not_retained_switches: if true then found switches are set to open, else it only checks and reports :return: updated original data """ - updated_switches = 0 + violated_switches = 0 original_models = load_opdm_objects_to_triplets(opdm_objects=original_data) not_retained_switches = original_models[(original_models['KEY'] == 'Switch.retained') & (original_models['VALUE'] == "false")][['ID']] @@ -86,20 +86,18 @@ def check_not_retained_switches_between_nodes(original_data, open_not_retained_s & (original_models['VALUE'] == 'false')] not_retained_closed = not_retained_switches.merge(closed_switches[['ID']], on='ID') terminals = original_models.type_tableview('Terminal').rename_axis('Terminal').reset_index() - terminals = terminals[['Terminal', - # 'ACDCTerminal.connected', - 'Terminal.ConductingEquipment', - 'Terminal.TopologicalNode']] + terminals = terminals[['Terminal', 'Terminal.ConductingEquipment', 'Terminal.TopologicalNode']] not_retained_terminals = (terminals.rename(columns={'Terminal.ConductingEquipment': 'ID'}) .merge(not_retained_closed, on='ID')) if not_retained_terminals.empty: - return original_data, updated_switches + return original_data, violated_switches + between_tn = ((not_retained_terminals.groupby('ID')[['Terminal.TopologicalNode']] .apply(lambda x: check_switch_terminals(x, 'Terminal.TopologicalNode'))) .reset_index(name='same_TN')) between_tn = between_tn[between_tn['same_TN']] if not between_tn.empty: - updated_switches = len(between_tn.index) + violated_switches = len(between_tn.index) logger.warning(f"Found {len(between_tn.index)} not retained switches between topological nodes") if open_not_retained_switches: logger.warning(f"Opening not retained switches") @@ -107,4 +105,4 @@ def check_not_retained_switches_between_nodes(original_data, open_not_retained_s open_switches.loc[:, 'VALUE'] = 'true' original_data = triplets.rdf_parser.update_triplet_from_triplet(original_data, open_switches) - return original_data, updated_switches + return original_data, violated_switches From 0d5c27a1225f63708ee238bf3f197090b5d42b68 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Mon, 30 Jun 2025 16:50:42 +0300 Subject: [PATCH 40/67] bugfix: handle zero division on HVDC scaling --- emf/model_merger/scaler.py | 1 + 1 file changed, 1 insertion(+) diff --git a/emf/model_merger/scaler.py b/emf/model_merger/scaler.py index 945793a0..6d80268d 100644 --- a/emf/model_merger/scaler.py +++ b/emf/model_merger/scaler.py @@ -118,6 +118,7 @@ def scale_balance(model: object, # Get pre-scale HVDC setpoints dangling_lines = get_network_elements(network, pp.network.ElementType.DANGLING_LINE, all_attributes=True) dangling_lines['power_factor'] = dangling_lines['boundary_q'] / dangling_lines['boundary_p'] # estimate the power factor + dangling_lines['power_factor'] = dangling_lines['power_factor'].fillna(0) # handle zero division dangling_lines['boundary_p'] = dangling_lines['boundary_p'] * -1 # invert boundary_p sign to match flow direction dangling_lines['boundary_q'] = dangling_lines['boundary_q'] * -1 # invert boundary_q sign to match flow direction prescale_hvdc_sp = dangling_lines[dangling_lines.isHvdc == 'true'][['lineEnergyIdentificationCodeEIC', 'boundary_p', 'boundary_q']] From a1df786c673a6f60e612adb49b88bed106d64920 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Tue, 1 Jul 2025 09:18:28 +0300 Subject: [PATCH 41/67] bugfix: circular import refactor: cleaned not releveant imports --- emf/model_merger/merge_functions.py | 32 ++++++----------------------- 1 file changed, 6 insertions(+), 26 deletions(-) diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index 0f37845b..ce2fe265 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -1,12 +1,6 @@ import json import math -import os.path -from io import BytesIO - -from triplets.cgmes_tools import get_metadata_from_FullModel -from triplets.rdf_parser import tableview_to_triplet, remove_triplet_from_triplet from xml.sax.expatreader import version - import pandas as pd import pypowsybl import logging @@ -14,17 +8,13 @@ import datetime import triplets import uuid -from uuid import uuid4 - import config from emf.common.config_parser import parse_app_properties from emf.common.integrations import elastic -# from emf.common.loadflow_tool import loadflow_settings from emf.model_merger import temporary from emf.common.helpers.time import parse_datetime from emf.common.helpers.loadflow import get_model_outages, get_network_elements from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, filename_from_opdm_metadata -from emf.model_merger.model_merger import MergedModel logger = logging.getLogger(__name__) @@ -157,7 +147,7 @@ def update_merged_model_sv(sv_data: bytes, opdm_object_meta: dict): updated_sv_id_map = {} for old_id in sv_data.query("KEY == 'Type' and VALUE == 'FullModel'").ID.unique(): if not is_valid_uuid(old_id): - new_id = str(uuid4()) + new_id = str(uuid.uuid4()) updated_sv_id_map[old_id] = new_id logger.warning(f"SV profile id {old_id} is not valid, assigning: {new_id}") sv_data = sv_data.replace(updated_sv_id_map) @@ -257,7 +247,7 @@ def create_updated_ssh(models_as_triplets: pd.DataFrame | list, # Generate new UUID for updated SSH updated_ssh_id_map = {} for OLD_ID in ssh_data.query("KEY == 'Type' and VALUE == 'FullModel'").ID.unique(): - NEW_ID = str(uuid4()) + NEW_ID = str(uuid.uuid4()) updated_ssh_id_map[OLD_ID] = NEW_ID logger.info(f"Assigned new UUID for updated SSH: {OLD_ID} -> {NEW_ID}") @@ -860,6 +850,8 @@ def run_post_merge_processing(input_models: list, if __name__ == "__main__": from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download + from emf.common.helpers.loadflow import load_network_model + from emf.common.loadflow_tool import loadflow_settings logging.basicConfig( format='%(levelname)-10s %(asctime)s.%(msecs)03d %(name)-30s %(funcName)-35s %(lineno)-5d: %(message)s', @@ -878,20 +870,8 @@ def run_post_merge_processing(input_models: list, valid_models = get_latest_models_and_download(time_horizon, scenario_date, valid=True) latest_boundary = get_latest_boundary() - merged_model = load_model(valid_models + [latest_boundary]) - - # TODO - run other LF if default fails - solved_model = run_lf(merged_model, loadflow_settings=loadflow_settings.CGM_DEFAULT) - - # TODO - get version dynamically form ELK - sv_data, ssh_data = create_sv_and_updated_ssh(solved_model, valid_models, time_horizon, version, merging_area, merging_entity, mas) - - # Fix SV - sv_data = fix_sv_shunts(sv_data, valid_models) - sv_data = fix_sv_tapsteps(sv_data, ssh_data) - - # Package to in memory zip files - serialized_data = export_to_cgmes_zip([ssh_data, sv_data]) + merged_model = load_network_model(valid_models + [latest_boundary]) + solved_model = pypowsybl.loadflow.run_ac(merged_model, loadflow_settings=loadflow_settings.CGM_DEFAULT) # Export to OPDM from emf.common.integrations.opdm import OPDM From 43aceb33b8970cb465cc8f71bdaeca0a84ba090a Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Tue, 1 Jul 2025 10:18:31 +0300 Subject: [PATCH 42/67] bugifx: merger return string instead python dict to be able to publish to quality queue --- emf/model_merger/model_merger.py | 2 +- emf/model_merger/worker.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 5c80a47c..4306e5b9 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -441,7 +441,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): logger.info(f"Merge task finished for model: {merged_model.name}") - return opdm_object_meta, properties + return json.dumps(opdm_object_meta), properties if __name__ == "__main__": diff --git a/emf/model_merger/worker.py b/emf/model_merger/worker.py index 83d8b787..f5dd2a51 100644 --- a/emf/model_merger/worker.py +++ b/emf/model_merger/worker.py @@ -2,6 +2,10 @@ from uuid import uuid4 from emf.common.logging import custom_logger +# Supress FutureWarnings from triplets library cause by pandas to_numeric errors ignore deprecation +import warnings +warnings.filterwarnings('ignore', category=FutureWarning) + # Initialize custom logger logger = logging.getLogger(__name__) elk_handler = custom_logger.initialize_custom_logger(extra={'worker': 'model-merger', 'worker_uuid': str(uuid4())}) From 7f31cbc6460d53dbdef38770e01d29c8ae2e2be1 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Tue, 1 Jul 2025 11:47:52 +0300 Subject: [PATCH 43/67] bugfix: validator failure when BDS received, now ignore it --- emf/model_validator/model_validator.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index ecc6bdfe..714a15fd 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -190,6 +190,12 @@ def handle(self, message: bytes, properties: dict, **kwargs): # Get network models data from object storage opdm_objects = [models.get_content(metadata=opdm_object) for opdm_object in opdm_metadata] + # Exclude BDS-type objects from validation + opdm_objects = [opdm_object for opdm_object in opdm_objects if opdm_object["opde:Object-Type"] != "BDS"] + if not opdm_objects: + logger.warning("No OPDM objects for validations or it was type of BDS, exiting validation") + return message, properties + # Get the latest boundary set for validation latest_boundary = models.get_latest_boundary() @@ -241,6 +247,7 @@ def handle(self, message: bytes, properties: dict, **kwargs): except Exception as error: logger.error(f"Models validator failed with exception: {error}", exc_info=True) + continue # Define model validity valid = all(report['validations'].values()) From 73eeb78aa4d6a1a596f1e34d56d45176380c15e1 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Tue, 1 Jul 2025 15:27:43 +0300 Subject: [PATCH 44/67] bugfix: removed pre temp fix parameter from task generator --- config/task_generator/manual_task_generator.properties | 1 - emf/task_generator/manual_worker.py | 1 - 2 files changed, 2 deletions(-) diff --git a/config/task_generator/manual_task_generator.properties b/config/task_generator/manual_task_generator.properties index 1b27cbfa..916c8161 100644 --- a/config/task_generator/manual_task_generator.properties +++ b/config/task_generator/manual_task_generator.properties @@ -19,7 +19,6 @@ RUN_SCALING = UPLOAD_TO_OPDM = UPLOAD_TO_MINIO = SEND_MERGE_REPORT = -PRE_TEMP_FIXES = POST_TEMP_FIXES = FORCE_OUTAGE_FIX = FIX_NET_INTERCHANGE2 = \ No newline at end of file diff --git a/emf/task_generator/manual_worker.py b/emf/task_generator/manual_worker.py index bdee904d..9bd22534 100644 --- a/emf/task_generator/manual_worker.py +++ b/emf/task_generator/manual_worker.py @@ -53,7 +53,6 @@ process_config_json[0]['runs'][0]['properties']['upload_to_opdm'] = UPLOAD_TO_OPDM process_config_json[0]['runs'][0]['properties']['upload_to_minio'] = UPLOAD_TO_MINIO process_config_json[0]['runs'][0]['properties']['send_merge_report'] = SEND_MERGE_REPORT -process_config_json[0]['runs'][0]['properties']['pre_temp_fixes'] = PRE_TEMP_FIXES process_config_json[0]['runs'][0]['properties']['post_temp_fixes'] = POST_TEMP_FIXES process_config_json[0]['runs'][0]['properties']['force_outage_fix'] = FORCE_OUTAGE_FIX From bf9901eb5def755882764f5d7337ab71db222b96 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Tue, 1 Jul 2025 16:12:39 +0300 Subject: [PATCH 45/67] bugfix: content reference generation from file name --- emf/common/helpers/opdm_objects.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/emf/common/helpers/opdm_objects.py b/emf/common/helpers/opdm_objects.py index 71c080e3..cf6aa868 100644 --- a/emf/common/helpers/opdm_objects.py +++ b/emf/common/helpers/opdm_objects.py @@ -129,10 +129,10 @@ def generate_opdm_object_content_reference_from_filename(file_name: str, opdm_ob Returns: str: Generated file path using the provided template and extracted metadata. """ - template = "{opdm_object_type}/{processType}/{modelingEntity}/{scenarioTime:%Y%m%d}/{scenarioTime:%H%M00}/{messageType}/{file_name}" + template = "{opdm_object_type}/{timeHorizon}/{sourcingActor}/{validFrom:%Y%m%d}/{validFrom:%H%M00}/{cgmesProfile}/{file_name}" - meta = {key.split(".")[-1]: value for key, value in get_metadata_from_file_name(file_name).items()} - meta["scenarioTime"] = parse_datetime(meta["scenarioTime"]) + meta = {key.split(":")[-1]: value for key, value in get_metadata_from_file_name(file_name).items()} + meta["validFrom"] = parse_datetime(meta["validFrom"]) meta["file_name"] = file_name meta["opdm_object_type"] = opdm_object_type From a11e1445fe5f56be1f4bf5fa2b63c54f9ca80d7d Mon Sep 17 00:00:00 2001 From: mr0321 <145195994+mr0321@users.noreply.github.com> Date: Wed, 2 Jul 2025 13:57:28 +0300 Subject: [PATCH 46/67] fixes ssh profile loading --- emf/model_merger/merge_functions.py | 28 +++++++++++++++++++--------- emf/model_merger/temporary.py | 2 +- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index ce2fe265..5347b0a4 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -156,14 +156,17 @@ def update_merged_model_sv(sv_data: bytes, opdm_object_meta: dict): def load_ssh(input_data: pd.DataFrame | list): + """ + Loads in ssh profiles from list of profiles or takes the slice from dataframe + :param input_data: list of profiles or dataframe + :return dataframe of ssh data + """ if not isinstance(input_data, pd.DataFrame): ssh_data = load_opdm_objects_to_triplets(input_data, "SSH") else: - ssh_files = input_data[input_data['VALUE'] == 'SSH'][['INSTANCE_ID']].drop_duplicates() - if ssh_files.empty: - ssh_files = (input_data[(input_data['KEY'] == 'label') & - (input_data['VALUE'].str.upper().str.contains('SSH'))][['INSTANCE_ID']] - .drop_duplicates()) + ssh_files = (input_data[(input_data['KEY'] == 'label') & + (input_data['VALUE'].str.upper().str.contains('SSH'))][['INSTANCE_ID']] + .drop_duplicates()) ssh_data = input_data.merge(ssh_files, on='INSTANCE_ID') ssh_data = triplets.cgmes_tools.update_FullModel_from_filename(ssh_data) return ssh_data @@ -171,14 +174,16 @@ def load_ssh(input_data: pd.DataFrame | list): def create_updated_ssh(models_as_triplets: pd.DataFrame | list, sv_data: pd.DataFrame, - opdm_object_meta: dict + opdm_object_meta: dict, + input_models: list = None, ): # TODO rewrite to use pypowsybl exported SSH ### SSH ## # Load original SSH data to created updated SSH - ssh_data = load_ssh(models_as_triplets) + ssh_file_data = input_models or models_as_triplets + ssh_data = load_ssh(ssh_file_data) # Update SSH Model.scenarioTime ssh_data.set_VALUE_at_KEY('Model.scenarioTime', opdm_object_meta['pmd:scenarioDate']) @@ -738,8 +743,12 @@ def check_all_kind_of_injections(cgm_sv_data, except AttributeError: logger.info(f"SSH profile doesn't contain data about {injection_name}") return cgm_ssh_data - injections_reduced = injections[[*fixed_fields, *fields_to_check.values()]] - original_injections_reduced = original_injections[[*fixed_fields, *fields_to_check.values()]] + try: + injections_reduced = injections[[*fixed_fields, *fields_to_check.values()]] + original_injections_reduced = original_injections[[*fixed_fields, *fields_to_check.values()]] + except KeyError as ke: + logger.info(f"{injection_name} tableview got error: {ke}") + return cgm_ssh_data injections_reduced = injections_reduced.merge(original_injections_reduced, on='ID', suffixes=('', '_org')) if terminals is None: terminals = (original_models.type_tableview('Terminal') @@ -791,6 +800,7 @@ def run_post_merge_processing(input_models: list, # Create update SSH sv_data, ssh_data, opdm_object_meta = create_updated_ssh(models_as_triplets=input_models_triplets, + input_models = input_models, sv_data=sv_data, opdm_object_meta=opdm_object_meta) fix_net_interchange_errors = False diff --git a/emf/model_merger/temporary.py b/emf/model_merger/temporary.py index 25371a6f..d7ece10d 100644 --- a/emf/model_merger/temporary.py +++ b/emf/model_merger/temporary.py @@ -154,7 +154,7 @@ def handle_igm_ssh_vs_cgm_ssh_error(network_pre_instance: pypowsybl.network.Netw lower_limit_violated = curve_generators[(curve_generators['min_p'] < curve_generators['curve_p_min'])] if not lower_limit_violated.empty: logger.warning(f"Updating min p from curve for {len(lower_limit_violated.index)} generators") - lower_limit_violated['min_p'] = lower_limit_violated['curve_p_min'] + lower_limit_violated.loc[:, 'min_p'] = lower_limit_violated['curve_p_min'] network_pre_instance.update_generators(lower_limit_violated[['id', 'min_p']].set_index('id')) # Solution 2: discard generator from participating From 39dbd007290a816adff9ec306d525de06e5915a1 Mon Sep 17 00:00:00 2001 From: mr0321 <145195994+mr0321@users.noreply.github.com> Date: Wed, 2 Jul 2025 20:01:50 +0300 Subject: [PATCH 47/67] for Kirchhoff 1st law --- config/model_validator/model_validator.properties | 2 +- emf/model_merger/temporary.py | 7 +++---- emf/model_validator/validator_functions.py | 5 ++++- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/config/model_validator/model_validator.properties b/config/model_validator/model_validator.properties index 0c21164e..756508db 100644 --- a/config/model_validator/model_validator.properties +++ b/config/model_validator/model_validator.properties @@ -8,4 +8,4 @@ ELK_ID_HASHING = True VALIDATION_LOAD_FLOW_SETTINGS = CGM_RELAXED_2 CHECK_NON_RETAINED_SWITCHES = False CHECK_KIRCHHOFF_FIRST_LAW = False -OPEN_NON_RETAINED_SWITCHES = False \ No newline at end of file +OPEN_NON_RETAINED_SWITCHES = True \ No newline at end of file diff --git a/emf/model_merger/temporary.py b/emf/model_merger/temporary.py index 25371a6f..0fd162a2 100644 --- a/emf/model_merger/temporary.py +++ b/emf/model_merger/temporary.py @@ -154,7 +154,7 @@ def handle_igm_ssh_vs_cgm_ssh_error(network_pre_instance: pypowsybl.network.Netw lower_limit_violated = curve_generators[(curve_generators['min_p'] < curve_generators['curve_p_min'])] if not lower_limit_violated.empty: logger.warning(f"Updating min p from curve for {len(lower_limit_violated.index)} generators") - lower_limit_violated['min_p'] = lower_limit_violated['curve_p_min'] + lower_limit_violated.loc[:, 'min_p'] = lower_limit_violated['curve_p_min'] network_pre_instance.update_generators(lower_limit_violated[['id', 'min_p']].set_index('id')) # Solution 2: discard generator from participating @@ -300,7 +300,7 @@ def set_paired_boundary_injections_to_zero(original_models, cgm_ssh_data): terminals = original_models.type_tableview("Terminal").reset_index()[['ID', 'Terminal.ConductingEquipment', 'Terminal.TopologicalNode']] - injections = original_models.type_tableview('EquivalentInjection').reset_index()[['ID', + injections = cgm_ssh_data.type_tableview('EquivalentInjection').reset_index()[['ID', # 'EquivalentInjection.p', # 'EquivalentInjection.q', # 'EquivalentInjection.regulationStatus' @@ -313,9 +313,8 @@ def set_paired_boundary_injections_to_zero(original_models, cgm_ssh_data): left_on="ID", right_on='Terminal.ConductingEquipment', suffixes=('_ConnectivityNode', '')) - paired_topological_injections = (topological_injections.groupby("Terminal.TopologicalNode") + paired_injections = (topological_injections.groupby("Terminal.TopologicalNode") .filter(lambda x: len(x) == 2)) - paired_injections = paired_topological_injections # Set terminal status updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) diff --git a/emf/model_validator/validator_functions.py b/emf/model_validator/validator_functions.py index bb598b3b..c6b3f92d 100644 --- a/emf/model_validator/validator_functions.py +++ b/emf/model_validator/validator_functions.py @@ -79,7 +79,10 @@ def check_not_retained_switches_between_nodes(original_data, open_not_retained_s :return: updated original data """ violated_switches = 0 - original_models = load_opdm_objects_to_triplets(opdm_objects=original_data) + if not isinstance(original_data, pandas.DataFrame): + original_models = load_opdm_objects_to_triplets(opdm_objects=original_data) + else: + original_models = original_data not_retained_switches = original_models[(original_models['KEY'] == 'Switch.retained') & (original_models['VALUE'] == "false")][['ID']] closed_switches = original_models[(original_models['KEY'] == 'Switch.open') From 6f6649a1c3bd32c630fc513fb8c9ba501b060c2e Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Thu, 3 Jul 2025 16:54:44 +0300 Subject: [PATCH 48/67] Model quality statistics key fix --- emf/model_quality/model_statistics.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/emf/model_quality/model_statistics.py b/emf/model_quality/model_statistics.py index 7c7ab824..e5ef8958 100644 --- a/emf/model_quality/model_statistics.py +++ b/emf/model_quality/model_statistics.py @@ -180,7 +180,7 @@ def get_system_metrics(data, tieflow_data=None, load_and_generation=None): losses_coefficient = losses / (abs(load) + abs(generation) + tieflow_abs_ei_p) if tieflow_abs_ei_p else None # Returning the computed metrics as a dictionary - return { + result = { 'total_load': load, 'generation': generation, 'losses': losses, @@ -189,9 +189,28 @@ def get_system_metrics(data, tieflow_data=None, load_and_generation=None): 'tieflow_np': tieflow_np, 'tieflow_acnp': tieflow_acnp, 'tieflow_hvdc': tieflow_hvdc, + } + # Fixes for ELK data storage + result = { + outer_key: { + key.replace('.p', '_p').replace('.q', '_q'): value + for key, value in inner_dict.items() + } if isinstance(inner_dict, dict) else inner_dict + for outer_key, inner_dict in result.items() } + tieflow_hvdc = [{**inner_key, 'eic': outer_key} for outer_key, inner_key in tieflow_hvdc.items()] + tieflow_hvdc = [ + {key.replace('.p', '_p').replace('.q', '_q'): value + for key, value in item.items() + } if isinstance(item, dict) else item + for item in tieflow_hvdc + ] + result['tieflow_hvdc'] = tieflow_hvdc + + return result + if __name__ == "__main__": import triplets import pandas From 31b52c7172b650668932208d16567a9257fe8e65 Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Thu, 3 Jul 2025 16:55:01 +0300 Subject: [PATCH 49/67] Model quality functions update --- emf/model_quality/model_quality.py | 253 +------------------------ emf/model_quality/quality_functions.py | 170 +++++++++++++++++ 2 files changed, 171 insertions(+), 252 deletions(-) diff --git a/emf/model_quality/model_quality.py b/emf/model_quality/model_quality.py index e6074652..dcb266a2 100644 --- a/emf/model_quality/model_quality.py +++ b/emf/model_quality/model_quality.py @@ -2,16 +2,13 @@ import pandas as pd import config import json -import pypowsybl as pp from emf.common.config_parser import parse_app_properties from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets from emf.common.integrations import elastic, minio_api from emf.common.integrations.object_storage import models -from emf.common.helpers import opdm_objects -from io import BytesIO -from zipfile import ZipFile, ZIP_DEFLATED from triplets.rdf_parser import load_all_to_dataframe from emf.model_quality.model_statistics import get_system_metrics +from emf.model_quality.quality_functions import generate_quality_report, process_zipped_cgm logger = logging.getLogger(__name__) @@ -37,7 +34,6 @@ def handle(self, message: bytes, properties: dict, **kwargs): unzipped = process_zipped_cgm(model_data) network= load_all_to_dataframe(unzipped) - # TODO test if IGM retrieving works as intended elif object_type == 'IGM': latest_boundary = models.get_latest_boundary() model_data = [models.get_content(metadata=opdm_object) for opdm_object in model_metadata] @@ -87,250 +83,3 @@ def handle(self, message: bytes, properties: dict, **kwargs): logger.error("Error, quality report generator failed, data not sent") return message, properties - - -def generate_quality_report(network, object_type, model_metadata): - - report = {} - - if object_type == "CGM" and model_metadata['merge_type'] == 'BA': - - # Check Kruonis generators - generators = network.type_tableview('SynchronousMachine').rename_axis('Terminal').reset_index() - kruonis_generators = generators[generators['IdentifiedObject.name'].str.contains('KHAE_G')] - - if not kruonis_generators.empty: - gen_count = kruonis_generators[kruonis_generators['RotatingMachine.p'] > 0].shape[0] - flag = gen_count < 3 - report.update({"kruonis_generators": gen_count, "kruonis_check": flag}) - else: - report.update({"kruonis_generators": None, "kruonis_check": False}) - - - # Check LT-PL crossborder flow - try: - control_areas = (network.type_tableview('ControlArea') - .rename_axis('ControlArea') - .reset_index())[['ControlArea', 'ControlArea.netInterchange', 'ControlArea.pTolerance', - 'IdentifiedObject.energyIdentCodeEic', 'IdentifiedObject.name']] - except KeyError: - control_areas = network.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() - ssh_areas = network.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() - control_areas = control_areas.merge(ssh_areas, on='ControlArea')[ - ['ControlArea', 'ControlArea.netInterchange', - 'ControlArea.pTolerance', - 'IdentifiedObject.energyIdentCodeEic', - 'IdentifiedObject.name']] - tie_flows = (network.type_tableview('TieFlow') - .rename_axis('TieFlow').rename(columns={'TieFlow.ControlArea': 'ControlArea', - 'TieFlow.Terminal': 'Terminal'}) - .reset_index())[['ControlArea', 'Terminal', 'TieFlow.positiveFlowIn']] - tie_flows = tie_flows.merge(control_areas[['ControlArea']], on='ControlArea') - try: - terminals = (network.type_tableview('Terminal') - .rename_axis('Terminal').reset_index())[['Terminal', 'ACDCTerminal.connected']] - except KeyError: - terminals = (network.type_tableview('Terminal') - .rename_axis('Terminal').reset_index())[['Terminal']] - tie_flows = tie_flows.merge(terminals, on='Terminal') - try: - power_flows_pre = (network.type_tableview('SvPowerFlow') - .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) - .reset_index())[['Terminal', 'SvPowerFlow.p']] - tie_flows = tie_flows.merge(power_flows_pre, on='Terminal', how='left') - except Exception: - logger.error(f"Was not able to get tie flows from original models") - power_flows_post = (network.type_tableview('SvPowerFlow') - .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) - .reset_index())[['Terminal', 'SvPowerFlow.p']] - - tie_flows = tie_flows.merge(power_flows_post, on='Terminal', how='left', - suffixes=('_pre', '_post')) - - # TODO double check correct limit value - # BORDER_LIMIT = 250 - # d_lines = network.get_dangling_lines(all_attributes=True) - # LT_PL_lines = d_lines[d_lines['name'].str.contains('Alytus-Elk')] - # if LT_PL_lines: - # flow_sum = LT_PL_lines['p'].sum() - # flag = flow_sum < BORDER_LIMIT - # report.update({"lt_pl_flow": flow_sum, "lt_pl_xborder_check": flag}) - # else: - - # TODO fix border flow - report.update({"lt_pl_flow": None, "lt_pl_xborder_check": False}) - - # TODO remake into Triplets - # Check cross-border line inconsistencies - # TODO log all line info - # pairing_keys = d_lines.groupby('pairing_key')['connected'].nunique() - # mismatch = len(pairing_keys[pairing_keys > 1].index.tolist()) - # flag = mismatch < 1 - # report.update({"xb_mismatch": mismatch, "xb_consitency_check": flag}) - - # TODO Check model outage mismatch with outage plan - # model_outages = pd.DataFrame(get_model_outages(network=network)) - - # TODO define IGM quality rules - elif object_type == "IGM": - report = model_metadata[0] - model_metadata[0].pop('opde:Component') - model_metadata[0].pop('opde:Dependencies') - report.update({"quality": "No Status"}) - - return report - - -def load_cgm(network, parameters): - - if bytes == type(network): - try: - network = unzip_zipped_profiles(BytesIO(network)) - except Exception as error: - logger.error(error) - network = pp.network.load_from_binary_buffer(network, parameters=parameters) - if str == type(network): - try: - network = unzip_zipped_profiles(BytesIO(network)) - network = pp.network.load_from_binary_buffer(network, parameters=parameters) - except Exception as error: - logger.error(error) - network = pp.network.load(network, parameters=parameters) - - return network - - -def process_zipped_cgm(zipped_bytes, processed=[]): - - with ZipFile(BytesIO(zipped_bytes)) as zf: - for name in zf.namelist(): - with zf.open(name) as file: - content = file.read() - if name.endswith('.zip'): - process_zipped_cgm(content) - elif name.endswith('.xml'): - file_object = BytesIO(content) - file_object.name = name - processed.append(file_object) - - return processed - - -def unzip_zipped_profiles(path_or_buffer: str): - """ - Extracts zipped profiles of a model. - Detects automatically if the zipped folder is read from file bath or BytesIO buffer. - :param path_or_buffer: path of zipped model or BytesIO buffer - :return: output_zip_buffer - """ - # Checks if path_or_buffer is string - if isinstance(path_or_buffer, str): - with open(path_or_buffer, 'rb') as original_zip_file: - original_zip_buffer = BytesIO(original_zip_file.read()) - - # Checks if path_or_buffer is BytesIO class - elif isinstance(path_or_buffer, BytesIO): - original_zip_buffer = path_or_buffer - else: - logger.error(f"Provided variable is nor string nor BytesIO object") - - output_zip_buffer = BytesIO() - - # Read the original zipped folder from provided buffer - with ZipFile(original_zip_buffer, 'r') as original_zip: - # Create a new zip file where we will store the unzipped XMLs - with ZipFile(output_zip_buffer, 'w', ZIP_DEFLATED) as new_zip: - # Iterate trough each file in the original zip - for file_name in original_zip.namelist(): - # Read the zipped xml file - with original_zip.open(file_name) as zipped_xml_file: - with ZipFile(zipped_xml_file) as zipped_xml: - # Extract each file and add it to the new zip - for xml_file_name in zipped_xml.namelist(): - xml_data = zipped_xml.read(xml_file_name) - # Add unzipped XML file to the new zip - new_zip.writestr(xml_file_name, xml_data) - - output_zip_buffer.seek(0) - output_zip_buffer.name = 'model' - - return output_zip_buffer - - -def query_elk_uap(index, time_horizon=None): - - from datetime import datetime - - logger.info(f"Retrieving outages from ELK index: '{index}'") - - # now represents the time of the run, in P0W case it should be current time - now = datetime.now() - now = now.strftime("%Y-%m-%dT%H:%M") + "Z" - - if time_horizon == 'WK': - merge_type_list = ['week'] - elif time_horizon == 'MO': - merge_type_list = ['week', 'month'] - elif time_horizon == 'YR': - merge_type_list = ['year'] - - query = { - "bool": { - "must": [ - {"exists": {"field": "name"}}, - {"terms": {"Merge": merge_type_list}}, - ], - "filter": [{"range": {"reportParsedDate": {"lte": now, "gte": "now-2w"}}}], - } - } - response = get_docs_by_query(index=index, query=query, size=10000, return_df=True) - result = pd.DataFrame() - - if not response.empty: - - # Get only latest report data - response['reportParsedDate'] = pd.to_datetime(response['reportParsedDate']) - response = response[response['reportParsedDate'] == response['reportParsedDate'].max()] - # Only keep latest outages - duplicated_outages = response[response.duplicated('eic', keep=False)] - latest_duplicate = duplicated_outages.groupby('eic')['date_of_last_change'].idxmax() - response = response.loc[response.index.isin(latest_duplicate) | ~response['eic'].duplicated(keep=False)] - - response = response[response['outage_type'].isin(['OUT'])] - - response = response.sort_values(by=['eic', 'start_date', 'end_date']).reset_index(drop=True) - last_end_time = {} - - # Remove outage duplicate if there is time overlap - for _, row in response.iterrows(): - eic = row['eic'] - start_time = row['start_date'] - end_time = row['end_date'] - - if eic not in last_end_time or start_time > last_end_time[eic]: - result = pd.concat([result, pd.DataFrame([row])], ignore_index=True) - last_end_time[eic] = end_time - - return result - - -if __name__ == "__main__": - # TESTING - import sys - from emf.common.integrations.opdm import OPDM - logging.basicConfig( - format='%(levelname)-10s %(asctime)s.%(msecs)03d %(name)-30s %(funcName)-35s %(lineno)-5d: %(message)s', - datefmt='%Y-%m-%dT%H:%M:%S', - level=logging.INFO, - handlers=[logging.StreamHandler(sys.stdout)] - ) - #logging.getLogger('powsybl').setLevel(1) - - opdm = OPDM() - # latest_boundary = opdm.get_latest_boundary() - available_models = opdm.get_latest_models_and_download(time_horizon='2D', - scenario_date="2025-06-12T09:30", - tso="AST") - - - opdm_metadata = json.loads(message) diff --git a/emf/model_quality/quality_functions.py b/emf/model_quality/quality_functions.py index e69de29b..e5e4e49e 100644 --- a/emf/model_quality/quality_functions.py +++ b/emf/model_quality/quality_functions.py @@ -0,0 +1,170 @@ +from io import BytesIO +from zipfile import ZipFile +import logging + +logger = logging.getLogger(__name__) + +def generate_quality_report(network, object_type, model_metadata): + + report = {} + + if object_type == "CGM" and model_metadata['merge_type'] == 'BA': + + # Check Kruonis generators + generators = network.type_tableview('SynchronousMachine').rename_axis('Terminal').reset_index() + kruonis_generators = generators[generators['IdentifiedObject.name'].str.contains('KHAE_G')] + + if not kruonis_generators.empty: + gen_count = kruonis_generators[kruonis_generators['RotatingMachine.p'] > 0].shape[0] + flag = gen_count < 3 + report.update({"kruonis_generators": gen_count, "kruonis_check": flag}) + else: + report.update({"kruonis_generators": None, "kruonis_check": False}) + + + # Check LT-PL crossborder flow + try: + control_areas = (network.type_tableview('ControlArea') + .rename_axis('ControlArea') + .reset_index())[['ControlArea', 'ControlArea.netInterchange', 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', 'IdentifiedObject.name']] + except KeyError: + control_areas = network.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + ssh_areas = network.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + control_areas = control_areas.merge(ssh_areas, on='ControlArea')[ + ['ControlArea', 'ControlArea.netInterchange', + 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', + 'IdentifiedObject.name']] + tie_flows = (network.type_tableview('TieFlow') + .rename_axis('TieFlow').rename(columns={'TieFlow.ControlArea': 'ControlArea', + 'TieFlow.Terminal': 'Terminal'}) + .reset_index())[['ControlArea', 'Terminal', 'TieFlow.positiveFlowIn']] + tie_flows = tie_flows.merge(control_areas[['ControlArea']], on='ControlArea') + try: + terminals = (network.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal', 'ACDCTerminal.connected']] + except KeyError: + terminals = (network.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal']] + tie_flows = tie_flows.merge(terminals, on='Terminal') + try: + power_flows_pre = (network.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + tie_flows = tie_flows.merge(power_flows_pre, on='Terminal', how='left') + except Exception: + logger.error(f"Was not able to get tie flows from original models") + power_flows_post = (network.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + + tie_flows = tie_flows.merge(power_flows_post, on='Terminal', how='left', + suffixes=('_pre', '_post')) + + # TODO double check correct limit value + # BORDER_LIMIT = 250 + # d_lines = network.get_dangling_lines(all_attributes=True) + # LT_PL_lines = d_lines[d_lines['name'].str.contains('Alytus-Elk')] + # if LT_PL_lines: + # flow_sum = LT_PL_lines['p'].sum() + # flag = flow_sum < BORDER_LIMIT + # report.update({"lt_pl_flow": flow_sum, "lt_pl_xborder_check": flag}) + # else: + + # TODO fix border flow + report.update({"lt_pl_flow": None, "lt_pl_xborder_check": False}) + + # TODO remake into Triplets + # Check cross-border line inconsistencies + # TODO log all line info + # pairing_keys = d_lines.groupby('pairing_key')['connected'].nunique() + # mismatch = len(pairing_keys[pairing_keys > 1].index.tolist()) + # flag = mismatch < 1 + # report.update({"xb_mismatch": mismatch, "xb_consitency_check": flag}) + + # TODO Check model outage mismatch with outage plan + # model_outages = pd.DataFrame(get_model_outages(network=network)) + + # TODO define IGM quality rules + elif object_type == "IGM": + report = model_metadata[0] + model_metadata[0].pop('opde:Component') + model_metadata[0].pop('opde:Dependencies') + report.update({"quality": "No Status"}) + + return report + + +# TODO temp function, later use common one +def query_elk_uap(index, time_horizon=None): + + from datetime import datetime + + logger.info(f"Retrieving outages from ELK index: '{index}'") + + # now represents the time of the run, in P0W case it should be current time + now = datetime.now() + now = now.strftime("%Y-%m-%dT%H:%M") + "Z" + + if time_horizon == 'WK': + merge_type_list = ['week'] + elif time_horizon == 'MO': + merge_type_list = ['week', 'month'] + elif time_horizon == 'YR': + merge_type_list = ['year'] + + query = { + "bool": { + "must": [ + {"exists": {"field": "name"}}, + {"terms": {"Merge": merge_type_list}}, + ], + "filter": [{"range": {"reportParsedDate": {"lte": now, "gte": "now-2w"}}}], + } + } + response = get_docs_by_query(index=index, query=query, size=10000, return_df=True) + result = pd.DataFrame() + + if not response.empty: + + # Get only latest report data + response['reportParsedDate'] = pd.to_datetime(response['reportParsedDate']) + response = response[response['reportParsedDate'] == response['reportParsedDate'].max()] + # Only keep latest outages + duplicated_outages = response[response.duplicated('eic', keep=False)] + latest_duplicate = duplicated_outages.groupby('eic')['date_of_last_change'].idxmax() + response = response.loc[response.index.isin(latest_duplicate) | ~response['eic'].duplicated(keep=False)] + + response = response[response['outage_type'].isin(['OUT'])] + + response = response.sort_values(by=['eic', 'start_date', 'end_date']).reset_index(drop=True) + last_end_time = {} + + # Remove outage duplicate if there is time overlap + for _, row in response.iterrows(): + eic = row['eic'] + start_time = row['start_date'] + end_time = row['end_date'] + + if eic not in last_end_time or start_time > last_end_time[eic]: + result = pd.concat([result, pd.DataFrame([row])], ignore_index=True) + last_end_time[eic] = end_time + + return result + + +def process_zipped_cgm(zipped_bytes, processed=[]): + + with ZipFile(BytesIO(zipped_bytes)) as zf: + for name in zf.namelist(): + with zf.open(name) as file: + content = file.read() + if name.endswith('.zip'): + process_zipped_cgm(content) + elif name.endswith('.xml'): + file_object = BytesIO(content) + file_object.name = name + processed.append(file_object) + + return processed From fc938e09f50d1f12333c2a8da3c206733aa5f291 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Fri, 4 Jul 2025 12:25:47 +0300 Subject: [PATCH 50/67] bugfix: merger return statement when models not available feature: RMQ consumer can check properties status flag and reject message refactor: simplified models variables naming in merger --- emf/common/integrations/rabbit.py | 18 +++++--- emf/model_merger/model_merger.py | 70 ++++++++++++++----------------- emf/model_merger/replacement.py | 4 +- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/emf/common/integrations/rabbit.py b/emf/common/integrations/rabbit.py index 495da745..7694156f 100644 --- a/emf/common/integrations/rabbit.py +++ b/emf/common/integrations/rabbit.py @@ -400,12 +400,20 @@ def _process_messages(self, basic_deliver, properties, body): # self.connection.close() # self.stop() + # Process message acknowledgment if ack: - # Publish message to next exchange/queue if provided and message successfully handled - if self.forward: - logger.info(f"Publishing message to exchange: {self.forward}") - self._channel.basic_publish(exchange=self.forward, routing_key="", body=body, properties=properties) - self.acknowledge_message(basic_deliver.delivery_tag) + # Check if properties has some status flag set from handler + _success = properties.headers.get('success', True) + if _success: + # Publish message to next exchange/queue if provided and message successfully handled + if self.forward: + logger.info(f"Publishing message to exchange: {self.forward}") + self._channel.basic_publish(exchange=self.forward, routing_key="", body=body, properties=properties) + self.acknowledge_message(basic_deliver.delivery_tag) + logger.info("Message acknowledged") + else: + logger.warning(f"Task rejected due to success flag set by handler: {_success}") + self._channel.basic_reject(basic_deliver.delivery_tag, requeue=False) def on_message(self, _unused_channel, basic_deliver, properties, body): """Invoked by pika when a message is delivered from RabbitMQ. The diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 4306e5b9..cae41a25 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -153,23 +153,29 @@ def handle(self, task_object: dict, properties: dict, **kwargs): force_outage_fix = task_properties['force_outage_fix'] # Collect valid models from ObjectStorage - downloaded_models = get_latest_models_and_download(time_horizon, scenario_datetime, valid=False, data_source='OPDM') + downloaded_models = get_latest_models_and_download(time_horizon=time_horizon, + scenario_date=scenario_datetime, + valid=True, + data_source='OPDM') latest_boundary = get_latest_boundary() # Filter out models that are not to be used in merge - filtered_models = merge_functions.filter_models(downloaded_models, included_models, excluded_models, filter_on='pmd:TSO') + models = merge_functions.filter_models(models=downloaded_models, + included_models=included_models, + excluded_models=excluded_models, + filter_on='pmd:TSO') - # Get additional models directly from Minio + # Get additional models from ObjectStorage if local import is configured if local_import_models: - additional_models_data = get_latest_models_and_download(time_horizon=time_horizon, - scenario_date=scenario_datetime, - valid=True, - data_source='PDN') - additional_models_data = merge_functions.filter_models(models=additional_models_data, - included_models=local_import_models, - filter_on='pmd:TSO') - - missing_local_import = [tso for tso in local_import_models if tso not in [model['pmd:TSO'] for model in additional_models_data]] + additional_models = get_latest_models_and_download(time_horizon=time_horizon, + scenario_date=scenario_datetime, + valid=True, + data_source='PDN') + additional_models = merge_functions.filter_models(models=additional_models, + included_models=local_import_models, + filter_on='pmd:TSO') + + missing_local_import = [tso for tso in local_import_models if tso not in [model['pmd:TSO'] for model in additional_models]] merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-pdn'} for tso in missing_local_import]) # Perform local replacement if configured @@ -186,34 +192,28 @@ def handle(self, task_object: dict, properties: dict, **kwargs): 'time_horizon': model['pmd:timeHorizon'], 'scenario_timestamp': model['pmd:scenarioDate']} for model in replacement_models_local] merged_model.replaced_entity.extend(replaced_entities_local) - additional_models_data.extend(replacement_models_local) + additional_models.extend(replacement_models_local) except Exception as error: logger.error(f"Failed to run replacement: {error} {error.with_traceback()}") else: - additional_models_data = [] - - # Check model validity and availability - valid_models = [model for model in filtered_models if model['valid']] - invalid_models = [model['pmd:TSO'] for model in filtered_models if model not in valid_models] - if invalid_models: - merged_model.excluded.extend([{'tso': tso, 'reason': 'invalid'} for tso in invalid_models]) + additional_models = [] # Check missing models for replacement if included_models: - missing_models = [model for model in included_models if model not in [model['pmd:TSO'] for model in valid_models]] + missing_models = [model for model in included_models if model not in [model['pmd:TSO'] for model in models]] if missing_models: merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-opdm'} for tso in missing_models]) else: if model_replacement: # Get TSOs who models are available in storage for replacement period available_tsos = get_tsos_available_in_storage() - valid_model_tsos = [model['pmd:TSO'] for model in valid_models] + valid_model_tsos = [model['pmd:TSO'] for model in models] # Need to ensure that excluded models by task configuration would not be taken in replacement context missing_models = [tso for tso in available_tsos if tso not in valid_model_tsos + excluded_models] else: missing_models = [] - # Run replacement on missing/invalid models + # Run replacement on missing models if model_replacement and missing_models: try: logger.info(f"Running replacement for missing models: {missing_models}") @@ -225,7 +225,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): 'scenario_timestamp': model['pmd:scenarioDate']} for model in replacement_models] merged_model.replaced_entity.extend(replaced_entities) - valid_models = valid_models + replacement_models + models.extend(replacement_models) merged_model.replaced = True else: merged_model.replaced = False @@ -233,34 +233,26 @@ def handle(self, task_object: dict, properties: dict, **kwargs): logger.error(f"Failed to run replacement: {error}") merged_model.replaced = False - # Store all relevant models for loading - valid_models = valid_models + additional_models_data - - # Return None if there are no models to be merged - if not valid_models: - logger.warning("Found no valid models to merge, returning None") - return None - - # Store models together with boundary - input_models = valid_models + [latest_boundary] + # Store models together with boundary set and check whether there are enough models to merge + input_models = models + additional_models + [latest_boundary] if len(input_models) < 2: - logger.warning("Found no models to merge, returning None") - return None + logger.warning("No valid models found for merging, exiting merge process") + properties.headers['success'] = False + return task_object, properties # Load network model and merge merge_start = datetime.datetime.now(datetime.UTC) merged_model.network = load_network_model(opdm_objects=input_models) merged_model.network_meta = attr_to_dict(instance=merged_model.network, sanitize_to_strings=True) - merged_model.included = [model['pmd:TSO'] for model in valid_models if model['pmd:TSO']] + merged_model.included = [model['pmd:TSO'] for model in input_models if model.get('pmd:TSO', None)] # Crosscheck replaced model outages with latest UAP if at least one Baltic model was replaced replaced_tso_list = [entity['tso'] for entity in merged_model.replaced_entity] - valid_tso_list = [tso['pmd:TSO'] for tso in valid_models] # Update model outages tso_list = [] if force_outage_fix: # force outage fix on all models if set - tso_list = valid_tso_list + tso_list = merged_model.included elif merging_area == 'BA' and any(tso in ['LITGRID', 'AST', 'ELERING'] for tso in replaced_tso_list): # by default do it on Baltic merge replaced models tso_list = replaced_tso_list if tso_list: # if not set force and not replaced BA then nothing to fix diff --git a/emf/model_merger/replacement.py b/emf/model_merger/replacement.py index 1f46cb57..43a4b8b6 100644 --- a/emf/model_merger/replacement.py +++ b/emf/model_merger/replacement.py @@ -64,13 +64,13 @@ def run_replacement(tso_list: list, replaced_tso = replacements['pmd:TSO'].unique().tolist() not_replaced = [model for model in unique_tsos_list if model not in replaced_tso] if not_replaced: - logger.error(f"Unable to find replacements within given replacement logic for TSO's: {not_replaced}") + logger.warning(f"Unable to find replacements within given replacement logic for TSO's: {not_replaced}") tso_missing = [model for model in tso_list if model not in unique_tsos_list] if tso_missing: logger.info(f"No replacement models found for TSO(s): {tso_missing}") else: - logger.error(f"No replacement models found, replacement list is empty") + logger.warning(f"No replacement models found, replacement list is empty") else: logger.info(f"No replacement models found in Elastic for TSO(s): {tso_list}") From 5b20a8aad94f4c6cd3b57be2f50c93d91f4e8637 Mon Sep 17 00:00:00 2001 From: VeikoAunapuu Date: Mon, 7 Jul 2025 10:50:42 +0300 Subject: [PATCH 51/67] VA_Cornercase when RMM is ran with empty list on included --- emf/task_generator/manual_worker.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/emf/task_generator/manual_worker.py b/emf/task_generator/manual_worker.py index 9bd22534..a76cdba4 100644 --- a/emf/task_generator/manual_worker.py +++ b/emf/task_generator/manual_worker.py @@ -1,5 +1,5 @@ import json -import os +import os, sys import config import logging from emf.task_generator.task_generator import generate_tasks @@ -40,6 +40,11 @@ timeframe_config_json = [time_conf] break +if "RMM" in RUN_TYPE and not(INCLUDED_TSO): + logger.error("RMM included TSOs can not be empty for the RUN_TYPE") + sys.exit("Issue with input, check the EMFOS logs for possible error") + + # Update process configuration from ENV variables if defined process_config_json[0]['runs'][0]['run_at'] = '* * * * *' process_config_json[0]['runs'][0]['properties']['included'] = [tso.strip() for tso in INCLUDED_TSO.split(',')] if INCLUDED_TSO else [] From 8d38d2c3c2e050400b300121f5620d949da8556c Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Mon, 7 Jul 2025 12:48:46 +0300 Subject: [PATCH 52/67] #320 dynamic validation settings relaxation --- .../model_validator.properties | 4 +- emf/model_validator/model_validator.py | 42 +++++++++++++------ 2 files changed, 32 insertions(+), 14 deletions(-) diff --git a/config/model_validator/model_validator.properties b/config/model_validator/model_validator.properties index 756508db..8485ce83 100644 --- a/config/model_validator/model_validator.properties +++ b/config/model_validator/model_validator.properties @@ -5,7 +5,9 @@ VALIDATION_ELK_INDEX = emfos-igm-validation METADATA_ELK_INDEX = emfos-opde-models ELK_ID_FROM_METADATA_FIELDS = opde:Id,data-source ELK_ID_HASHING = True -VALIDATION_LOAD_FLOW_SETTINGS = CGM_RELAXED_2 +ENABLE_DYNAMIC_VALIDATION_SETTINGS = True +VALIDATION_LOAD_FLOW_SETTINGS = CGM_DEFAULT +VALIDATION_LOAD_FLOW_SETTINGS_PRIORITY = CGM_DEFAULT,CGM_RELAXED_1,CGM_RELAXED_2,CGM_RELAXED_3_CUSTOM CHECK_NON_RETAINED_SWITCHES = False CHECK_KIRCHHOFF_FIRST_LAW = False OPEN_NON_RETAINED_SWITCHES = True \ No newline at end of file diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index 714a15fd..64f65923 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -57,11 +57,27 @@ def __init__(self, network: pp.network, network_triplets: pd.DataFrame): def validate_loadflow(self): """Validate load flow convergence""" - logger.info(f"Solving load flow with settings: {VALIDATION_LOAD_FLOW_SETTINGS}") - loadflow_report = pp.report.Reporter() - loadflow_result = pp.loadflow.run_ac(network=self.network, - parameters=self.loadflow_parameters, - reporter=loadflow_report) + # Set starting point of lf settings priority list + if json.loads(ENABLE_DYNAMIC_VALIDATION_SETTINGS.lower()): + settings_list = [param.strip() for param in VALIDATION_LOAD_FLOW_SETTINGS_PRIORITY.split(",")] + settings_priority = next((i for i, value in enumerate(settings_list) if value == VALIDATION_LOAD_FLOW_SETTINGS), None) + settings_list = settings_list[settings_priority:] + else: + settings_list = [VALIDATION_LOAD_FLOW_SETTINGS] + + # Run powerflow, relaxing lf settings after each diverging result + for lf_settings in settings_list: + loadflow_parameters = getattr(loadflow_settings, lf_settings) + logger.info(f"Solving load flow with settings: '{lf_settings}'") + loadflow_report = pp.report.Reporter() + loadflow_result = pp.loadflow.run_ac(network=self.network, + parameters=loadflow_parameters, + reporter=loadflow_report) + + if loadflow_result[0].status_text == 'Converged': + break + else: + logger.warning(f"Failed to solve load flow with settings: {lf_settings}") # Parsing aggregated results self.report['components'] = len(loadflow_result) @@ -79,7 +95,7 @@ def validate_loadflow(self): self.report['loadflow'] = component_results # Validation status - self.report['loadflow_parameters'] = VALIDATION_LOAD_FLOW_SETTINGS + self.report['loadflow_parameters'] = lf_settings self.report['validations']['loadflow'] = True if main_component.status.value == 0 else False def validate_network_elements(self): @@ -199,7 +215,7 @@ def handle(self, message: bytes, properties: dict, **kwargs): # Get the latest boundary set for validation latest_boundary = models.get_latest_boundary() - logger.info(f"Validation parameters used: {VALIDATION_LOAD_FLOW_SETTINGS}") + # logger.info(f"Validation parameters used: {VALIDATION_LOAD_FLOW_SETTINGS}") # Run network model validations for opdm_object in opdm_objects: @@ -288,19 +304,19 @@ def handle(self, message: bytes, properties: dict, **kwargs): level=logging.INFO, handlers=[logging.StreamHandler(sys.stdout)] ) - #logging.getLogger('powsybl').setLevel(1) opdm = OPDM() - latest_boundary = opdm.get_latest_boundary() - available_models = opdm.get_latest_models_and_download(time_horizon='1D', - scenario_date="2025-06-16T09:30", - tso="LITGRID") + latest_boundary = models.get_latest_boundary() + available_models = models.get_latest_models_and_download(time_horizon='1D', + scenario_date="20250706T0930", + tso="AST", + data_source='opdm') validated_models = [] # Validate models for model in available_models: network_triplets = load_opdm_objects_to_triplets(opdm_objects=[model, latest_boundary]) - network = load_model(opdm_objects=[model, latest_boundary]) + network = load_network_model(opdm_objects=[model, latest_boundary]) post_lf_validation = PostLFValidator(network=network, network_triplets=network_triplets) post_lf_validation.run_validation() From 1d98ce7432f42a56aa396e18b2437b57cd7054c8 Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Mon, 7 Jul 2025 14:17:44 +0300 Subject: [PATCH 53/67] #320 merge settings auto relaxation --- config/cgm_worker/merger.properties | 4 ++- .../model_validator.properties | 6 ++-- emf/model_merger/model_merger.py | 29 ++++++++++++++----- emf/model_validator/model_validator.py | 2 +- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/config/cgm_worker/merger.properties b/config/cgm_worker/merger.properties index 36dda2ea..b1a05de4 100644 --- a/config/cgm_worker/merger.properties +++ b/config/cgm_worker/merger.properties @@ -7,5 +7,7 @@ OUTPUT_MINIO_BUCKET = opde-confidential-models OUTPUT_MINIO_FOLDER = EMFOS/RMM MERGE_REPORT_ELK_INDEX = emfos-merge-reports OPDE_MODELS_ELK_INDEX = emfos-opde-models -MERGE_LOAD_FLOW_SETTINGS = CGM_RELAXED_3_CUSTOM +ENABLE_DYNAMIC_MERGE_SETTINGS = True +MERGE_LOAD_FLOW_SETTINGS = CGM_DEFAULT +MERGE_LOAD_FLOW_SETTINGS_PRIORITY = CGM_DEFAULT,CGM_RELAXED_1,CGM_RELAXED_2,CGM_RELAXED_3_CUSTOM REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION = True \ No newline at end of file diff --git a/config/model_validator/model_validator.properties b/config/model_validator/model_validator.properties index 8485ce83..8193b364 100644 --- a/config/model_validator/model_validator.properties +++ b/config/model_validator/model_validator.properties @@ -5,9 +5,9 @@ VALIDATION_ELK_INDEX = emfos-igm-validation METADATA_ELK_INDEX = emfos-opde-models ELK_ID_FROM_METADATA_FIELDS = opde:Id,data-source ELK_ID_HASHING = True -ENABLE_DYNAMIC_VALIDATION_SETTINGS = True -VALIDATION_LOAD_FLOW_SETTINGS = CGM_DEFAULT -VALIDATION_LOAD_FLOW_SETTINGS_PRIORITY = CGM_DEFAULT,CGM_RELAXED_1,CGM_RELAXED_2,CGM_RELAXED_3_CUSTOM +ENABLE_DYNAMIC_VALIDATION_SETTINGS = False +VALIDATION_LOAD_FLOW_SETTINGS = IGM_VALIDATION +VALIDATION_LOAD_FLOW_SETTINGS_PRIORITY = IGM_VALIDATION,CGM_DEFAULT,CGM_RELAXED_1,CGM_RELAXED_2,CGM_RELAXED_3_CUSTOM CHECK_NON_RETAINED_SWITCHES = False CHECK_KIRCHHOFF_FIRST_LAW = False OPEN_NON_RETAINED_SWITCHES = True \ No newline at end of file diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index cae41a25..853ee129 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -79,11 +79,26 @@ def __init__(self): @staticmethod def run_loadflow(merged_model): - # report = pypowsybl.report.Reporter() - result = pypowsybl.loadflow.run_ac(network=merged_model.network, - parameters=getattr(loadflow_settings, MERGE_LOAD_FLOW_SETTINGS), - # reporter=loadflow_report, - ) + # Set starting point of lf settings priority list + if json.loads(ENABLE_DYNAMIC_MERGE_SETTINGS.lower()): + settings_list = [param.strip() for param in MERGE_LOAD_FLOW_SETTINGS_PRIORITY.split(",")] + settings_priority = next((i for i, value in enumerate(settings_list) if value == MERGE_LOAD_FLOW_SETTINGS), None) + settings_list = settings_list[settings_priority:] + else: + settings_list = [MERGE_LOAD_FLOW_SETTINGS] + + for lf_settings in settings_list: + logger.info(f"Solving load flow with settings: '{lf_settings}'") + # report = pypowsybl.report.Reporter() + result = pypowsybl.loadflow.run_ac(network=merged_model.network, + parameters=getattr(loadflow_settings, lf_settings), + # reporter=loadflow_report, + ) + if result[0].status_text == 'Converged': + logger.info(f"Model successfully converged with loadflow settings: '{lf_settings}'") + break + else: + logger.warning(f"Failed to solve load flow with settings:' {lf_settings}'") result_dict = [attr_to_dict(island) for island in result] # Modify all nested objects to native data types @@ -102,6 +117,7 @@ def run_loadflow(merged_model): # merged_model.loadflow = str(loadflow_report) merged_model.loadflow = [island for island in result_dict if island['reference_bus_id']] merged_model.loadflow_status = result[0].status.name # store main island loadflow status + merged_model.loadflow_settings = lf_settings return merged_model @@ -408,7 +424,6 @@ def handle(self, task_object: dict, properties: dict, **kwargs): logger.debug(task) # Update merged model attributes - merged_model.loadflow_settings = MERGE_LOAD_FLOW_SETTINGS merged_model.duration_s = (end_time - merge_start).total_seconds() merged_model.content_reference = merged_model_object.name @@ -466,7 +481,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): "job_period_start": "2024-05-24T22:00:00+00:00", "job_period_end": "2024-05-25T06:00:00+00:00", "task_properties": { - "timestamp_utc": "2025-06-30T12:30:00+00:00", + "timestamp_utc": "2025-07-06T12:30:00+00:00", "merge_type": "BA", "merging_entity": "BALTICRCC", "included": ["LITGRID", "AST", "ELERING", "PSE"], diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index 64f65923..accee1cd 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -77,7 +77,7 @@ def validate_loadflow(self): if loadflow_result[0].status_text == 'Converged': break else: - logger.warning(f"Failed to solve load flow with settings: {lf_settings}") + logger.warning(f"Failed to solve load flow with settings: '{lf_settings}'") # Parsing aggregated results self.report['components'] = len(loadflow_result) From 6c6feb5bf60814739b11ee84eeca29383eec00ce Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Wed, 9 Jul 2025 10:41:43 +0300 Subject: [PATCH 54/67] minor fine tune of merged pull requests with bugfix to forward LF settings to scaler --- emf/model_merger/model_merger.py | 11 +++++------ emf/model_validator/model_validator.py | 13 +++++++------ emf/task_generator/manual_worker.py | 9 +++++---- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 853ee129..cecbec96 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -88,17 +88,16 @@ def run_loadflow(merged_model): settings_list = [MERGE_LOAD_FLOW_SETTINGS] for lf_settings in settings_list: - logger.info(f"Solving load flow with settings: '{lf_settings}'") + logger.info(f"Solving loadflow with settings: {lf_settings}") # report = pypowsybl.report.Reporter() result = pypowsybl.loadflow.run_ac(network=merged_model.network, parameters=getattr(loadflow_settings, lf_settings), # reporter=loadflow_report, ) if result[0].status_text == 'Converged': - logger.info(f"Model successfully converged with loadflow settings: '{lf_settings}'") break else: - logger.warning(f"Failed to solve load flow with settings:' {lf_settings}'") + logger.warning(f"Failed to solve loadflow with settings: {lf_settings}") result_dict = [attr_to_dict(island) for island in result] # Modify all nested objects to native data types @@ -290,7 +289,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): # TODO - run other LF if default fails # Run loadflow on merged model merged_model = self.run_loadflow(merged_model=merged_model) - logger.info(f"Loadflow status of main island: {merged_model.loadflow[0]['status_text']}") + logger.info(f"Loadflow status of main island: {merged_model.loadflow_status} [settings: {merged_model.loadflow_settings}]") # Perform scaling if model_scaling: @@ -312,13 +311,13 @@ def handle(self, task_object: dict, properties: dict, **kwargs): merged_model = scaler.scale_balance(model=merged_model, ac_schedules=ac_schedules, dc_schedules=dc_schedules, - lf_settings=getattr(loadflow_settings, MERGE_LOAD_FLOW_SETTINGS)) + lf_settings=getattr(loadflow_settings, merged_model.loadflow_settings)) except Exception as e: logger.error(e) merged_model.scaled = False else: logger.warning(f"Schedule reference data not available: {schedule_time_horizon} for {schedule_start}") - logger.warning(f"Model schedule scaling not performed") + logger.warning(f"Network model schedule scaling not performed") merged_model.scaled = False # Record main merging process end diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index accee1cd..bcafd83b 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -65,19 +65,20 @@ def validate_loadflow(self): else: settings_list = [VALIDATION_LOAD_FLOW_SETTINGS] - # Run powerflow, relaxing lf settings after each diverging result + # Run loadflow, relaxing settings after each diverging result for lf_settings in settings_list: loadflow_parameters = getattr(loadflow_settings, lf_settings) - logger.info(f"Solving load flow with settings: '{lf_settings}'") - loadflow_report = pp.report.Reporter() + logger.info(f"Solving loadflow with settings: {lf_settings}") + # loadflow_report = pp.report.Reporter() loadflow_result = pp.loadflow.run_ac(network=self.network, parameters=loadflow_parameters, - reporter=loadflow_report) + # reporter=loadflow_report, + ) if loadflow_result[0].status_text == 'Converged': break else: - logger.warning(f"Failed to solve load flow with settings: '{lf_settings}'") + logger.warning(f"Failed to solve loadflow with settings: {lf_settings}") # Parsing aggregated results self.report['components'] = len(loadflow_result) @@ -88,7 +89,7 @@ def validate_loadflow(self): # TODO currently storing only main island results main_component = loadflow_result[0] component_results = attr_to_dict(main_component) - logger.info(f"Loadflow status: {component_results['status_text']}") + logger.info(f"Loadflow status: {main_component.status.name} [settings: {lf_settings}]") component_results['status'] = component_results.get('status').value component_results['distributed_active_power'] = 0.0 if math.isnan(component_results['distributed_active_power'])\ else component_results['distributed_active_power'] diff --git a/emf/task_generator/manual_worker.py b/emf/task_generator/manual_worker.py index a76cdba4..697d5530 100644 --- a/emf/task_generator/manual_worker.py +++ b/emf/task_generator/manual_worker.py @@ -1,5 +1,6 @@ import json -import os, sys +import os +import sys import config import logging from emf.task_generator.task_generator import generate_tasks @@ -24,13 +25,13 @@ # Based on RUN_TYPE get process timeframe config -break_top=False +break_top = False for merge_index, merge_type in enumerate(process_config_json): for run in merge_type.get("runs", []): if RUN_TYPE in run["@id"]: process_config_json.pop(merge_index) process_config_json[0]['runs'] = [run] - break_top=True + break_top = True break if break_top: break @@ -41,7 +42,7 @@ break if "RMM" in RUN_TYPE and not(INCLUDED_TSO): - logger.error("RMM included TSOs can not be empty for the RUN_TYPE") + logger.error(f"RMM included TSOs can not be empty for the run type: {RUN_TYPE}") sys.exit("Issue with input, check the EMFOS logs for possible error") From 4d0c26d33634c07499e0de37c4138650db6ae585 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Wed, 9 Jul 2025 11:03:28 +0300 Subject: [PATCH 55/67] bump to pypowsybl 1.11.2 --- Pipfile | 2 +- Pipfile.lock | 68 +++++++++++++++++++++++----------------------- docker/Dockerfile | 5 ++-- requirements.txt | Bin 1757 -> 3568 bytes 4 files changed, 37 insertions(+), 38 deletions(-) diff --git a/Pipfile b/Pipfile index 05391717..17e9c60e 100644 --- a/Pipfile +++ b/Pipfile @@ -6,7 +6,7 @@ name = "pypi" [packages] pandas = "2.2.2" numpy = "2.0.0" -pypowsybl = "1.9.0" +pypowsybl = "*" aniso8601 = "*" lxml = "*" edx = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 404d5b38..43f45cc2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -457,11 +457,11 @@ }, "networkx": { "hashes": [ - "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", - "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f" + "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", + "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037" ], - "markers": "python_version >= '3.10'", - "version": "==3.4.2" + "markers": "python_version >= '3.11'", + "version": "==3.5" }, "numpy": { "hashes": [ @@ -578,11 +578,11 @@ }, "prettytable": { "hashes": [ - "sha256:61d5c68f04a94acc73c7aac64f0f380f5bed4d2959d59edc6e4cbb7a0e7b55c4", - "sha256:b804b8d51db23959b96b329094debdbbdf10c8c3aa75958c5988cfd7f78501dd" + "sha256:3c64b31719d961bf69c9a7e03d0c1e477320906a98da63952bc6698d6164ff57", + "sha256:b5eccfabb82222f5aa46b798ff02a8452cf530a352c31bddfa29be41242863aa" ], "markers": "python_version >= '3.9'", - "version": "==3.14.0" + "version": "==3.16.0" }, "pycparser": { "hashes": [ @@ -632,30 +632,30 @@ }, "pypowsybl": { "hashes": [ - "sha256:06a9ac540861f1bf271838c6151a61d35886b3f89ab23a29dc88d1f42392d6b6", - "sha256:1015e4d0acc51634fe476a31a05e4b5827afe6cef2a5aa1a9b9ff97bba90c1a1", - "sha256:1f7106f3bdaf751446383ddd784c695210161889f77e1fe0e1994ae9616a3e24", - "sha256:4f06d9007395b92802a1f7ce891de915265bcff8aa6ac8baca4a93436e091ca3", - "sha256:4fd7e29245b913c6bd643ceb46d66a516a074147498412843922b3a56d1072f4", - "sha256:5db2abac09aec5feb0b6fc8d769fc788187ea3ac657e39c36d82238c59186621", - "sha256:7d6db29334de8bb9218db6919ff7ac0e9b3c159d957d229ac2a975f3229c7454", - "sha256:804b8a46989f303f5ef8851c2ba798fc7cf7b157b4ff3a9eb6a4e74bd117a5fa", - "sha256:8728f39fdc94cc22bae4e4189f0875cf0e2d7b8711c012522322a784a6d6a2b1", - "sha256:8974e5879450fa8505dccf5defe32d5b43b77b83e1a1237dbe7dd962db568095", - "sha256:8dc084734ddb34be73dd5ce727160a914eeb50d856ea112d3827a11ff3d6fb18", - "sha256:900ca4d292ee471d067e3552930dd29ba8e972ca49717644c142e03937fae0d3", - "sha256:9254f48c09e19fdef00f776f09c636976ee7373be7a2ef7209709993dadca40f", - "sha256:949752533c16126eaadec6ff2f9d9a1f101d5a31e145c4cb11fac5a106e3cb94", - "sha256:ce052c2b2bcf8fd044cab613bd220806d6d74cce3b4cf56842f6d0aa35115579", - "sha256:d76af6ee9ad031889e3356758c2d9e313c6cd5b625d6e72389a6c976e644f75a", - "sha256:d79df2aa4d6b4146aed7bc35be6092090740fa7c6a3e2dab09366dcc6e2619a2", - "sha256:d9ba5291da53b9c6660e3b80b9980486b23dc9976768c0e8f66338cc3a3a0ed3", - "sha256:f2bfb79da425483122b352f3a0ae636c1831dd25bcfdcad8354cda083c7e391a", - "sha256:f37cb5a3e4f74c06a642af9e33f0e07a7f7a6546bb5c223f51cddd73fc4eb73f" + "sha256:06eee912c5fd05ca0d21393a350f2355c22953767dd43d8f3defa51adfbdb858", + "sha256:23e628b62521dd46aa8e4b93aee5a58c366ace971d69bbd09775d1294c4d1f85", + "sha256:255c9e72a09c89fd0d99f98f02d34ca10f1d64108bd57059d4c7917c541db8b2", + "sha256:52cc308a921ae9221d093bcb01ebe7fc219753e91907d1d8b1c5910bc3da765b", + "sha256:54cda0f1b8115c01587a2fa73ecb76ce7b74435b60f017bed412da02bae7d75f", + "sha256:6325324a13fe2689063d2c96679d46517a05bc609358b6e502243f5572e14daf", + "sha256:6b575ff9b2ccb9d6dcb44f57d6824d00cf37293e31a7051eac27a03e7bf85708", + "sha256:776561013a13a113954b8aaa930b96935a8267897f2c570a1bd5045ce11c45fc", + "sha256:7da64de9f09906c0fba3d8caa9a5c32de9637c5b1c52016af885a49c4a7a1a4a", + "sha256:95dc568bb50db2be8e5fa5b3c5bdbf232f71db98ddd1869fd3a8429c9c73f948", + "sha256:a03ef2b6244388bdd869a2486646119ec2aa3efa49fc236603ab1ab161eb5ef9", + "sha256:a192d50800e4b08c5a04d72835243ffb64318b7652713fd4875ea318e51b6174", + "sha256:b1074d3ca73715521541200f2b8f75c0fd6450e26789013968022f4be2b12531", + "sha256:b5b94ed9c45d281130c7ab93258434e6e7856d1cb01c29b38e4ea67d4cdf5237", + "sha256:b7f52b7d01fb9b99245dc50a89322549eb953e74c3ad1ae95842cab648adc87a", + "sha256:c462c20b9a7ea1aef9930ec2c909fd54df7f269d7aa931628c1d68dc8b9209f8", + "sha256:c8cc64683e7c05a147dd1216aa912f301befcf2c752085b09ea5e074a06592ae", + "sha256:def9daf2c3410d49dda9ba97e4afea2254a6bf85c348c24db66ac471527ff99e", + "sha256:eaf992676631769b0dd1b009cfa041f001eacf1142ea4c087a6928942ba1fdb0", + "sha256:f8afb4255eedc060e81637fdc8943a3be60e52da241367030e2923632a586a08" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.9.0" + "version": "==1.11.2" }, "python-dateutil": { "hashes": [ @@ -667,11 +667,11 @@ }, "pytz": { "hashes": [ - "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57", - "sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e" + "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", + "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00" ], "index": "pypi", - "version": "==2025.1" + "version": "==2025.2" }, "requests": { "hashes": [ @@ -755,11 +755,11 @@ }, "tzdata": { "hashes": [ - "sha256:24894909e88cdb28bd1636c6887801df64cb485bd593f2fd83ef29075a81d694", - "sha256:7e127113816800496f027041c570f50bcd464a020098a3b6b199517772303639" + "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", + "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9" ], "markers": "python_version >= '2'", - "version": "==2025.1" + "version": "==2025.2" }, "urllib3": { "hashes": [ diff --git a/docker/Dockerfile b/docker/Dockerfile index d88a6ebd..99a54441 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,6 +1,6 @@ # Define base container image (platform has to be defined if using with ARM64 architecture) -FROM python:3.11 as emfos-base -#FROM --platform=linux/amd64 python:3.11 as emfos-base +FROM python:3.11-slim as emfos-base +#FROM --platform=linux/amd64 python:3.11-slim as emfos-base # Set working directory WORKDIR /src @@ -11,7 +11,6 @@ RUN pip install --upgrade setuptools pip uv pipenv # Install needed modules COPY Pipfile . COPY Pipfile.lock . -#COPY requirements.txt . #RUN PIPENV_VENV_IN_PROJECT=1 pipenv install --deploy #RUN pipenv install --system --deploy RUN pipenv requirements > requirements.txt diff --git a/requirements.txt b/requirements.txt index 5a31a1b98e370656c993cd5394a21af31689589e..894be74ef3d9b407c73d22999dd2931648f1170a 100644 GIT binary patch literal 3568 zcmd5;U2~IA5Zz~I{118aMQVUHNpm3I6G?MHF27Rer+9Mm_TPNWNgtNZ#q5 zDgK)OQ??_yf`?i%eO|)`qe>9Z8J^+ZCU@S54`VWW=L79|^b5_fMU*`vV?;>1Gu&w~ zE*|MI%FH^JDW0Jy&$_%9;AF0O=g2<-veZL`cO_!K)hvhz+2oy^U~?&3tS%qrl~EeE z3-}=O1$0HbLcp!SNPwN>OERkDUBU%&9pLJaV#|lq=As*Zts$Tb1DU$_orZv_96%iWwWydkQ z%6ITaWqRDE3&vd|n^tzp=}o5ve!U39DTWTL3To61JMKZfS@#OZ!lPfoQU-ewkY{K2 z4A>*as?;^UFh{DDU+K~AFh#mYdbxo$+M*YtajHK&RJ~m9{gUSn6|+=)D@0j9-4>_! z;!xXOIcr!Vcded5bDSVN;~2HEOKpZKu*cwl8YHre)_a+uIy_$5+~^720*?%Lbrf-; z6%`UO;~~<|*(beso_FfD31fwdX^;sqcb%{T!j;mq*-sLYs+oOe3wDdn2eb7#g*}+P z`L&^C9l)ldTSbPfCfX4btyXIG2lC1_{=Q;Ut=W)Mk}7xf?puv?KJj`Wa(%z}cP-Vp z0?Jp2ssyUesGW)b-$hhL!3iKy3 z+(JoY%qN(s>Wp2HoxH{*A7RxV>vH^Jj>N6+!G_z=uGV6jw5h;OjAOlrD93o+A}`LI z1+oZEXC0;NL?_zY3%3Vr-#$~+I2pY&bk*0QyXjvh(u(DxRBm{i zmh)5^$BEUSN(9QEmOtI1(8Ec}nuA{!+NwmRuP7>)f{!HSS~L%H5BhZ^X%+kZ-{od- zieJ2b*MA591!x!)X*RV44fZrzDIbv0O!%X5#!IdWlAXpROY#Fim1OIbfQZ; zyEaP;y!}>ITf_Oc7Zz?mlJ|C_EyfC6$nDYt5$PSZAvp`=c2M+o*=F j=?ZKRkZXG6j^WHN5>R;7I$TFA2O$3~+*UDS#H{}WAuJw5 From 4bbc92e46ffe32e9c833fbaa7890a871bd4a38ee Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Wed, 9 Jul 2025 12:06:59 +0300 Subject: [PATCH 56/67] #324 removed local replacement from config --- config/task_generator/manual_task_generator.properties | 1 - emf/model_merger/model_merger.py | 4 +--- emf/task_generator/manual_worker.py | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) diff --git a/config/task_generator/manual_task_generator.properties b/config/task_generator/manual_task_generator.properties index 916c8161..aa26369f 100644 --- a/config/task_generator/manual_task_generator.properties +++ b/config/task_generator/manual_task_generator.properties @@ -14,7 +14,6 @@ TASK_REFERENCE_TIME = TASK_VERSION = TASK_MERGING_ENTITY = RUN_REPLACEMENT = -RUN_REPLACEMENT_LOCAL = RUN_SCALING = UPLOAD_TO_OPDM = UPLOAD_TO_MINIO = diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index cecbec96..075dcb32 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -159,7 +159,6 @@ def handle(self, task_object: dict, properties: dict, **kwargs): mas = task_properties["mas"] version = task_properties["version"] model_replacement = task_properties["replacement"] - model_replacement_local = task_properties["replacement_local"] model_scaling = task_properties["scaling"] model_upload_to_opdm = task_properties["upload_to_opdm"] model_upload_to_minio = task_properties["upload_to_minio"] @@ -194,7 +193,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-pdn'} for tso in missing_local_import]) # Perform local replacement if configured - if model_replacement_local and missing_local_import: + if model_replacement and missing_local_import: try: logger.info(f"Running replacement for local storage missing models: {missing_local_import}") replacement_models_local = run_replacement(tso_list=missing_local_import, @@ -492,7 +491,6 @@ def handle(self, task_object: dict, properties: dict, **kwargs): "post_temp_fixes": "True", "fix_net_interchange2": "True", "replacement": "True", - "replacement_local": "True", "scaling": "True", "upload_to_opdm": "False", "upload_to_minio": "True", diff --git a/emf/task_generator/manual_worker.py b/emf/task_generator/manual_worker.py index 697d5530..fdb7baa2 100644 --- a/emf/task_generator/manual_worker.py +++ b/emf/task_generator/manual_worker.py @@ -54,7 +54,6 @@ process_config_json[0]['runs'][0]['properties']['fix_net_interchange2'] = FIX_NET_INTERCHANGE2 process_config_json[0]['runs'][0]['properties']['version'] = TASK_VERSION process_config_json[0]['runs'][0]['properties']['replacement'] = RUN_REPLACEMENT -process_config_json[0]['runs'][0]['properties']['replacement_local'] = RUN_REPLACEMENT_LOCAL process_config_json[0]['runs'][0]['properties']['scaling'] = RUN_SCALING process_config_json[0]['runs'][0]['properties']['upload_to_opdm'] = UPLOAD_TO_OPDM process_config_json[0]['runs'][0]['properties']['upload_to_minio'] = UPLOAD_TO_MINIO From 5d96a44e6e84f92234a017ebbe882348bea6f089 Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Wed, 9 Jul 2025 14:24:37 +0300 Subject: [PATCH 57/67] #326 Added dynamic replacement query length based on time horizon --- config/cgm_worker/replacement.properties | 3 +-- config/cgm_worker/replacement_conf.json | 3 +++ emf/model_merger/replacement.py | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/config/cgm_worker/replacement.properties b/config/cgm_worker/replacement.properties index 7713cf6f..5b2cf7e4 100644 --- a/config/cgm_worker/replacement.properties +++ b/config/cgm_worker/replacement.properties @@ -1,3 +1,2 @@ [MAIN] -REPLACEMENT_CONFIG = config/cgm_worker/replacement_config.json -QUERY_FILTER = now-2w \ No newline at end of file +REPLACEMENT_CONFIG = config/cgm_worker/replacement_config.json \ No newline at end of file diff --git a/config/cgm_worker/replacement_conf.json b/config/cgm_worker/replacement_conf.json index 3772c501..02e60153 100644 --- a/config/cgm_worker/replacement_conf.json +++ b/config/cgm_worker/replacement_conf.json @@ -7,6 +7,9 @@ "timeHorizon": { "Request_list": ["ID", "1D", "2D", "WK", "1D", "MO", "YR"] }, + "replacementLength": { + "Request_list": ["2w", "2w", "2w", "2w", "2w", "1m", "4m"] + }, "month_ahead": { "hours": ["07:30", "08:30", "09:30", "06:30", "05:30", "04:30"], "business_type": ["1D"] diff --git a/emf/model_merger/replacement.py b/emf/model_merger/replacement.py index 43a4b8b6..3572bd69 100644 --- a/emf/model_merger/replacement.py +++ b/emf/model_merger/replacement.py @@ -35,9 +35,11 @@ def run_replacement(tso_list: list, replacements = pd.DataFrame() # TODO time horizon exclusion logic + exclude available models from query # TODO put in query object type if CGM metadata objects will be stored + # Get replacement length by time horizon + query_filter = 'now-' + config["replacementLength"]["Request_list"][config["timeHorizon"]["Request_list"].index(time_horizon)] + # Query for available replacement models query = {"pmd:TSO.keyword": tso_list, "valid": True, "data-source": data_source} - body = query_data(query, QUERY_FILTER) - model_df = pd.DataFrame(body) + model_df = pd.DataFrame(query_data(query, query_filter)) # Set scenario dat to UTC if not model_df.empty: From e226c944c3d3cb2aa0721ffb2a2449a2c2ca1a2e Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Wed, 9 Jul 2025 16:05:55 +0300 Subject: [PATCH 58/67] Added opdm metadata to merger message properties header --- emf/model_merger/model_merger.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 075dcb32..c29e99b5 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -441,6 +441,10 @@ def handle(self, task_object: dict, properties: dict, **kwargs): except Exception as error: logger.error(f"Failed to create merge report: {error}") + # Append message headers with OPDM root metadata + extracted_meta = {key: value for key, value in opdm_object_meta.items() if isinstance(value, str)} + properties['headers'].update(extracted_meta) + # Stop Trace self.elk_logging_handler.stop_trace() From 9b7eebcf8925d44502efc3ebe63511263d7e78f2 Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Wed, 9 Jul 2025 16:34:40 +0300 Subject: [PATCH 59/67] quickfix: properties header update --- emf/model_merger/model_merger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index c29e99b5..3cf834c7 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -443,7 +443,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): # Append message headers with OPDM root metadata extracted_meta = {key: value for key, value in opdm_object_meta.items() if isinstance(value, str)} - properties['headers'].update(extracted_meta) + properties.headers = extracted_meta # Stop Trace self.elk_logging_handler.stop_trace() From 8354e7983dcb6aba53772fae31ff3a363dfc5965 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Wed, 9 Jul 2025 16:53:52 +0300 Subject: [PATCH 60/67] dependencies: triplets to 0.0.10 and opdmapi to 0.1.3 --- Pipfile.lock | 507 ++++++++++++++++++++++------------------------- requirements.txt | Bin 3568 -> 3608 bytes 2 files changed, 233 insertions(+), 274 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 43f45cc2..5ac6eea6 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -18,11 +18,11 @@ "default": { "aniso8601": { "hashes": [ - "sha256:1d2b7ef82963909e93c4f24ce48d4de9e66009a21bf1c1e1c85bdd0812fe412f", - "sha256:72e3117667eedf66951bb2d93f4296a56b94b078a8a95905a052611fb3f1b973" + "sha256:25488f8663dd1528ae1f54f94ac1ea51ae25b4d531539b8bc707fed184d16845", + "sha256:eb19717fd4e0db6de1aab06f12450ab92144246b257423fe020af5748c0cb89e" ], "index": "pypi", - "version": "==9.0.1" + "version": "==10.0.1" }, "argon2-cffi": { "hashes": [ @@ -61,19 +61,19 @@ }, "attrs": { "hashes": [ - "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", - "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2" + "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", + "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b" ], - "markers": "python_version >= '3.7'", - "version": "==24.2.0" + "markers": "python_version >= '3.8'", + "version": "==25.3.0" }, "certifi": { "hashes": [ - "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", - "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" + "sha256:c1d2ec05395148ee10cf672ffc28cd37ea0ab0d99f9cc74c43e588cbd111b079", + "sha256:d842783a14f8fdd646895ac26f719a061408834473cfc10203f6a575beb15d39" ], - "markers": "python_version >= '3.6'", - "version": "==2024.8.30" + "markers": "python_version >= '3.7'", + "version": "==2025.7.9" }, "cffi": { "hashes": [ @@ -150,99 +150,101 @@ }, "charset-normalizer": { "hashes": [ - "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", - "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", - "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", - "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", - "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", - "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", - "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", - "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", - "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", - "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", - "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", - "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", - "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", - "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", - "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", - "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", - "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", - "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", - "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", - "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", - "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", - "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", - "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", - "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", - "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", - "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", - "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", - "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", - "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", - "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", - "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", - "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", - "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", - "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", - "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", - "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", - "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", - "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", - "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", - "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", - "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", - "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", - "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", - "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", - "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", - "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", - "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", - "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", - "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", - "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", - "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", - "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", - "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", - "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", - "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", - "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", - "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", - "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", - "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", - "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", - "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", - "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", - "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", - "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", - "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", - "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", - "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", - "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", - "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", - "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", - "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", - "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", - "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", - "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", - "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", - "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", - "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", - "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", - "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", - "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", - "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", - "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", - "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", - "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", - "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", - "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", - "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", - "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", - "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", - "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" + "sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4", + "sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45", + "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7", + "sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0", + "sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7", + "sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d", + "sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d", + "sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0", + "sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184", + "sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db", + "sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b", + "sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64", + "sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b", + "sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8", + "sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff", + "sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344", + "sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58", + "sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e", + "sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471", + "sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148", + "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a", + "sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836", + "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e", + "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", + "sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c", + "sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1", + "sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01", + "sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366", + "sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58", + "sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5", + "sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c", + "sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2", + "sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a", + "sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597", + "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b", + "sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5", + "sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb", + "sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f", + "sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0", + "sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941", + "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", + "sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86", + "sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7", + "sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7", + "sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455", + "sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6", + "sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4", + "sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0", + "sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3", + "sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1", + "sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6", + "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981", + "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c", + "sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980", + "sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645", + "sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7", + "sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12", + "sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa", + "sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd", + "sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef", + "sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f", + "sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2", + "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d", + "sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5", + "sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02", + "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3", + "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd", + "sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e", + "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214", + "sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd", + "sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a", + "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c", + "sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681", + "sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba", + "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f", + "sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a", + "sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28", + "sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691", + "sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82", + "sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a", + "sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027", + "sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7", + "sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518", + "sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf", + "sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b", + "sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9", + "sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544", + "sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da", + "sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509", + "sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f", + "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a", + "sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f" ], - "markers": "python_full_version >= '3.7.0'", - "version": "==3.3.2" + "markers": "python_version >= '3.7'", + "version": "==3.4.2" }, "croniter": { "hashes": [ @@ -280,163 +282,120 @@ }, "idna": { "hashes": [ - "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac", - "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603" + "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", + "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" ], "markers": "python_version >= '3.6'", - "version": "==3.8" + "version": "==3.10" }, "isodate": { "hashes": [ - "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96", - "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" + "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", + "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6" ], - "version": "==0.6.1" + "markers": "python_version >= '3.7'", + "version": "==0.7.2" }, "lxml": { "hashes": [ - "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e", - "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229", - "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3", - "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5", - "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70", - "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15", - "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002", - "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd", - "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22", - "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf", - "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22", - "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832", - "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727", - "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e", - "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30", - "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f", - "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f", - "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51", - "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4", - "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de", - "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875", - "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42", - "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e", - "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6", - "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391", - "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc", - "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b", - "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237", - "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4", - "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86", - "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f", - "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a", - "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8", - "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f", - "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903", - "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03", - "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e", - "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99", - "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7", - "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab", - "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d", - "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22", - "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492", - "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b", - "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3", - "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be", - "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469", - "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f", - "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a", - "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c", - "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a", - "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4", - "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94", - "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442", - "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b", - "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84", - "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c", - "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9", - "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1", - "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be", - "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367", - "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e", - "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21", - "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa", - "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16", - "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d", - "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe", - "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83", - "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba", - "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040", - "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763", - "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8", - "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff", - "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2", - "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a", - "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b", - "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce", - "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c", - "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577", - "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8", - "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71", - "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512", - "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540", - "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f", - "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2", - "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a", - "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce", - "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e", - "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2", - "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27", - "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1", - "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d", - "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1", - "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330", - "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920", - "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99", - "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff", - "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18", - "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff", - "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c", - "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179", - "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080", - "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19", - "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d", - "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70", - "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32", - "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a", - "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2", - "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79", - "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3", - "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5", - "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f", - "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d", - "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3", - "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b", - "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753", - "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9", - "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957", - "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033", - "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb", - "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656", - "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab", - "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b", - "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d", - "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd", - "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859", - "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11", - "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c", - "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a", - "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005", - "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654", - "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80", - "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e", - "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec", - "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7", - "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965", - "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945", - "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8" + "sha256:013090383863b72c62a702d07678b658fa2567aa58d373d963cca245b017e065", + "sha256:032e65120339d44cdc3efc326c9f660f5f7205f3a535c1fdbf898b29ea01fb72", + "sha256:048a930eb4572829604982e39a0c7289ab5dc8abc7fc9f5aabd6fbc08c154e93", + "sha256:04d67ceee6db4bcb92987ccb16e53bef6b42ced872509f333c04fb58a3315256", + "sha256:059c4cbf3973a621b62ea3132934ae737da2c132a788e6cfb9b08d63a0ef73f9", + "sha256:0e32698462aacc5c1cf6bdfebc9c781821b7e74c79f13e5ffc8bfe27c42b1abf", + "sha256:1676b56d48048a62ef77a250428d1f31f610763636e0784ba67a9740823988ca", + "sha256:17f090a9bc0ce8da51a5632092f98a7e7f84bca26f33d161a98b57f7fb0004ca", + "sha256:185efc2fed89cdd97552585c624d3c908f0464090f4b91f7d92f8ed2f3b18f54", + "sha256:1fa377b827ca2023244a06554c6e7dc6828a10aaf74ca41965c5d8a4925aebb4", + "sha256:2181e4b1d07dde53986023482673c0f1fba5178ef800f9ab95ad791e8bdded6a", + "sha256:219e0431ea8006e15005767f0351e3f7f9143e793e58519dc97fe9e07fae5563", + "sha256:21db1ec5525780fd07251636eb5f7acb84003e9382c72c18c542a87c416ade03", + "sha256:246b40f8a4aec341cbbf52617cad8ab7c888d944bfe12a6abd2b1f6cfb6f6082", + "sha256:2793a627e95d119e9f1e19720730472f5543a6d84c50ea33313ce328d870f2dd", + "sha256:2930aa001a3776c3e2601cb8e0a15d21b8270528d89cc308be4843ade546b9ab", + "sha256:2ae06fbab4f1bb7db4f7c8ca9897dc8db4447d1a2b9bee78474ad403437bcc29", + "sha256:2b4790b558bee331a933e08883c423f65bbcd07e278f91b2272489e31ab1e2b4", + "sha256:2cfcf84f1defed7e5798ef4f88aa25fcc52d279be731ce904789aa7ccfb7e8d2", + "sha256:2dd1cc3ea7e60bfb31ff32cafe07e24839df573a5e7c2d33304082a5019bcd58", + "sha256:2f34687222b78fff795feeb799a7d44eca2477c3d9d3a46ce17d51a4f383e32e", + "sha256:310b719b695b3dd442cdfbbe64936b2f2e231bb91d998e99e6f0daf991a3eba3", + "sha256:34190a1ec4f1e84af256495436b2d196529c3f2094f0af80202947567fdbf2e7", + "sha256:35bc626eec405f745199200ccb5c6b36f202675d204aa29bb52e27ba2b71dea8", + "sha256:36531f81c8214e293097cd2b7873f178997dae33d3667caaae8bdfb9666b76c0", + "sha256:390240baeb9f415a82eefc2e13285016f9c8b5ad71ec80574ae8fa9605093cd7", + "sha256:40442e2a4456e9910875ac12951476d36c0870dcb38a68719f8c4686609897c4", + "sha256:4337e4aec93b7c011f7ee2e357b0d30562edd1955620fdd4aeab6aacd90d43c5", + "sha256:43cfbb7db02b30ad3926e8fceaef260ba2fb7df787e38fa2df890c1ca7966c3b", + "sha256:43fe5af2d590bf4691531b1d9a2495d7aab2090547eaacd224a3afec95706d76", + "sha256:46b9ed911f36bfeb6338e0b482e7fe7c27d362c52fde29f221fddbc9ee2227e7", + "sha256:4d23854ecf381ab1facc8f353dcd9adeddef3652268ee75297c1164c987c11dc", + "sha256:4d6036c3a296707357efb375cfc24bb64cd955b9ec731abf11ebb1e40063949f", + "sha256:4eb114a0754fd00075c12648d991ec7a4357f9cb873042cc9a77bf3a7e30c9db", + "sha256:4ee56288d0df919e4aac43b539dd0e34bb55d6a12a6562038e8d6f3ed07f9e36", + "sha256:51a5e4c61a4541bd1cd3ba74766d0c9b6c12d6a1a4964ef60026832aac8e79b3", + "sha256:522fe7abb41309e9543b0d9b8b434f2b630c5fdaf6482bee642b34c8c70079c8", + "sha256:54c4855eabd9fc29707d30141be99e5cd1102e7d2258d2892314cf4c110726c3", + "sha256:5592401cdf3dc682194727c1ddaa8aa0f3ddc57ca64fd03226a430b955eab6f6", + "sha256:58ffd35bd5425c3c3b9692d078bf7ab851441434531a7e517c4984d5634cd65b", + "sha256:5967fe415b1920a3877a4195e9a2b779249630ee49ece22021c690320ff07452", + "sha256:5fcd7d3b1d8ecb91445bd71b9c88bdbeae528fefee4f379895becfc72298d181", + "sha256:63b634facdfbad421d4b61c90735688465d4ab3a8853ac22c76ccac2baf98d97", + "sha256:690b20e3388a7ec98e899fd54c924e50ba6693874aa65ef9cb53de7f7de9d64a", + "sha256:6da7cd4f405fd7db56e51e96bff0865b9853ae70df0e6720624049da76bde2da", + "sha256:7488a43033c958637b1a08cddc9188eb06d3ad36582cebc7d4815980b47e27ef", + "sha256:74e748012f8c19b47f7d6321ac929a9a94ee92ef12bc4298c47e8b7219b26541", + "sha256:78718d8454a6e928470d511bf8ac93f469283a45c354995f7d19e77292f26108", + "sha256:7bf61bc4345c1895221357af8f3e89f8c103d93156ef326532d35c707e2fb19d", + "sha256:7da298e1659e45d151b4028ad5c7974917e108afb48731f4ed785d02b6818994", + "sha256:84ef591495ffd3f9dcabffd6391db7bb70d7230b5c35ef5148354a134f56f2be", + "sha256:85b14a4689d5cff426c12eefe750738648706ea2753b20c2f973b2a000d3d261", + "sha256:8a2e76efbf8772add72d002d67a4c3d0958638696f541734304c7f28217a9cab", + "sha256:8a78d6c9168f5bcb20971bf3329c2b83078611fbe1f807baadc64afc70523b3a", + "sha256:8cb26f51c82d77483cdcd2b4a53cda55bbee29b3c2f3ddeb47182a2a9064e4eb", + "sha256:8db5dc617cb937ae17ff3403c3a70a7de9df4852a046f93e71edaec678f721d0", + "sha256:9ab542c91f5a47aaa58abdd8ea84b498e8e49fe4b883d67800017757a3eb78e8", + "sha256:9da022c14baeec36edfcc8daf0e281e2f55b950249a455776f0d1adeeada4734", + "sha256:9f4b481b6cc3a897adb4279216695150bbe7a44c03daba3c894f49d2037e0a24", + "sha256:a52a4704811e2623b0324a18d41ad4b9fabf43ce5ff99b14e40a520e2190c851", + "sha256:a55da151d0b0c6ab176b4e761670ac0e2667817a1e0dadd04a01d0561a219349", + "sha256:a674c0948789e9136d69065cc28009c1b1874c6ea340253db58be7622ce6398f", + "sha256:ae74f7c762270196d2dda56f8dd7309411f08a4084ff2dfcc0b095a218df2e06", + "sha256:afd27d8629ae94c5d863e32ab0e1d5590371d296b87dae0a751fb22bf3685741", + "sha256:b2d71cdefda9424adff9a3607ba5bbfc60ee972d73c21c7e3c19e71037574816", + "sha256:b34339898bb556a2351a1830f88f751679f343eabf9cf05841c95b165152c9e7", + "sha256:b372d10d17a701b0945f67be58fae4664fd056b85e0ff0fbc1e6c951cdbc0512", + "sha256:b3c98d5b24c6095e89e03d65d5c574705be3d49c0d8ca10c17a8a4b5201b72f5", + "sha256:b8dd6dd0e9c1992613ccda2bcb74fc9d49159dbe0f0ca4753f37527749885c25", + "sha256:bd5913b4972681ffc9718bc2d4c53cde39ef81415e1671ff93e9aa30b46595e7", + "sha256:c0b5fa5eda84057a4f1bbb4bb77a8c28ff20ae7ce211588d698ae453e13c6281", + "sha256:c16304bba98f48a28ae10e32a8e75c349dd742c45156f297e16eeb1ba9287a1f", + "sha256:c24b8efd9c0f62bad0439283c2c795ef916c5a6b75f03c17799775c7ae3c0c9e", + "sha256:c2a5e8d207311a0170aca0eb6b160af91adc29ec121832e4ac151a57743a1e1e", + "sha256:c352fc8f36f7e9727db17adbf93f82499457b3d7e5511368569b4c5bd155a922", + "sha256:c86df1c9af35d903d2b52d22ea3e66db8058d21dc0f59842ca5deb0595921141", + "sha256:c907516d49f77f6cd8ead1322198bdfd902003c3c330c77a1c5f3cc32a0e4d16", + "sha256:ca50bd612438258a91b5b3788c6621c1f05c8c478e7951899f492be42defc0da", + "sha256:d18a25b19ca7307045581b18b3ec9ead2b1db5ccd8719c291f0cd0a5cec6cb81", + "sha256:d4f0c66df4386b75d2ab1e20a489f30dc7fd9a06a896d64980541506086be1f1", + "sha256:d6e200909a119626744dd81bae409fc44134389e03fbf1d68ed2a55a2fb10991", + "sha256:d7ae472f74afcc47320238b5dbfd363aba111a525943c8a34a1b657c6be934c3", + "sha256:db0efd6bae1c4730b9c863fc4f5f3c0fa3e8f05cae2c44ae141cb9dfc7d091dc", + "sha256:dbdd7679a6f4f08152818043dbb39491d1af3332128b3752c3ec5cebc0011a72", + "sha256:e0b1520ef900e9ef62e392dd3d7ae4f5fa224d1dd62897a792cf353eb20b6cae", + "sha256:e2030956cf4886b10be9a0285c6802e078ec2391e1dd7ff3eb509c2c95a69b76", + "sha256:e35e8aaaf3981489f42884b59726693de32dabfc438ac10ef4eb3409961fd402", + "sha256:e380e85b93f148ad28ac15f8117e2fd8e5437aa7732d65e260134f83ce67911b", + "sha256:edf6e4c8fe14dfe316939711e3ece3f9a20760aabf686051b537a7562f4da91a", + "sha256:f3389924581d9a770c6caa4df4e74b606180869043b9073e2cec324bad6e306e", + "sha256:f64ccf593916e93b8d36ed55401bb7fe9c7d5de3180ce2e10b08f82a8f397316", + "sha256:f720a14aa102a38907c6d5030e3d66b3b680c3e6f6bc95473931ea3c00c59967", + "sha256:f8d19565ae3eb956d84da3ef367aa7def14a2735d05bd275cd54c0301f0d0d6c", + "sha256:f97487996a39cb18278ca33f7be98198f278d0bc3c5d0fd4d7b3d63646ca3c8a" ], "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==5.3.0" + "markers": "python_version >= '3.8'", + "version": "==6.0.0" }, "minio": { "hashes": [ @@ -517,11 +476,11 @@ }, "opdm-api": { "hashes": [ - "sha256:5191a42a228bdfeb5025dfd8c63d59128e3e637990952a002a79ddb5aae4c04e", - "sha256:7d95536756d5fb09dc17cceed3ad1b2db95aaa04a6bf3ed5ee3e168b4600e4ce" + "sha256:91913afc4a74e5fdeea5ba4c3442382342e0031b6384dbb05568691ff9495ef2", + "sha256:e5e1884d848190232ec62b7467e80451286a0c3c845faa87fbbf68837cc12c35" ], "index": "pypi", - "version": "==0.1.0" + "version": "==0.1.3" }, "pandas": { "hashes": [ @@ -570,11 +529,11 @@ }, "platformdirs": { "hashes": [ - "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee", - "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3" + "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc", + "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4" ], - "markers": "python_version >= '3.8'", - "version": "==4.2.2" + "markers": "python_version >= '3.9'", + "version": "==4.3.8" }, "prettytable": { "hashes": [ @@ -675,12 +634,12 @@ }, "requests": { "hashes": [ - "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", - "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" + "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", + "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==2.32.3" + "version": "==2.32.4" }, "requests-file": { "hashes": [ @@ -739,11 +698,11 @@ }, "triplets": { "hashes": [ - "sha256:3eb48071d266b890efe94a57ef93addda6acfd6cc5b6cd6cf9923bbe59dddbe3", - "sha256:73c116c51d23248ee7af5772625f222ea1489408fa1930f60ae748db9e913e68" + "sha256:2ab66c9036a20a741cccbf1170fa446947ded88665736cab8f7f1c2a8d0f6235", + "sha256:edf08b458a7afc2001e0db71e0efa6198363bdeba6f4b07ef46fe99a292687f7" ], "index": "pypi", - "version": "==0.0.7" + "version": "==0.0.10" }, "typing-extensions": { "hashes": [ @@ -763,11 +722,11 @@ }, "urllib3": { "hashes": [ - "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472", - "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168" + "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", + "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" ], - "markers": "python_version >= '3.8'", - "version": "==2.2.2" + "markers": "python_version >= '3.9'", + "version": "==2.5.0" }, "wcwidth": { "hashes": [ @@ -778,19 +737,19 @@ }, "xmltodict": { "hashes": [ - "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56", - "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852" + "sha256:201e7c28bb210e374999d1dde6382923ab0ed1a8a5faeece48ab525b7810a553", + "sha256:20cc7d723ed729276e808f26fb6b3599f786cbc37e06c65e192ba77c40f20aac" ], - "markers": "python_version >= '3.4'", - "version": "==0.13.0" + "markers": "python_version >= '3.6'", + "version": "==0.14.2" }, "zeep": { "hashes": [ - "sha256:6754feb4c34a4b6d65fbc359252bf6654dcce3937bf1d95aae4402a60a8f5939", - "sha256:72093acfdb1d8360ed400869b73fbf1882b95c4287f798084c42ee0c1ff0e425" + "sha256:a637aa7eedb6330bb27e8c94c5233ddf23553904323adf9398f8cf5025acb216", + "sha256:f45385e9e1b09d5550e0f51ab9fa7c6842713cab7194139372fd82a99c56a06e" ], - "markers": "python_version >= '3.7'", - "version": "==4.2.1" + "markers": "python_version >= '3.8'", + "version": "==4.3.1" } }, "develop": { diff --git a/requirements.txt b/requirements.txt index 894be74ef3d9b407c73d22999dd2931648f1170a..7f22789bfd96f5b9dd0a036d2558ef632ddb77b6 100644 GIT binary patch delta 184 zcmew$Jws+f0J9;3!NkDlljpHmO!njEm|Vdq!D-5%$6yY`mXjSBEhk@LtY9>mtjK0O zxqy*t^9-g%jEsg88x1EbFmg;TVi99BhstpX<}g$+}Mx)6Sd5tH3;NaScICM^*^SKvD9 Date: Wed, 9 Jul 2025 17:12:23 +0300 Subject: [PATCH 61/67] including supporting tools to base image --- docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 99a54441..a53e68e4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -6,6 +6,7 @@ FROM python:3.11-slim as emfos-base WORKDIR /src # Install tools to set up python environment +RUN apt-get update && apt-get install -y --no-install-recommends procps && rm -rf /var/lib/apt/lists/* RUN pip install --upgrade setuptools pip uv pipenv # Install needed modules From 0c38b46094467b44f773eacdc0f02cc5caaf1008 Mon Sep 17 00:00:00 2001 From: Gintautas Poderys <91978404+gpoderys@users.noreply.github.com> Date: Wed, 9 Jul 2025 17:36:59 +0300 Subject: [PATCH 62/67] Quality module CGM updated metadata --- emf/model_quality/model_quality.py | 6 +++--- emf/model_quality/model_statistics.py | 6 +++++- emf/model_quality/quality_functions.py | 8 ++++++-- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/emf/model_quality/model_quality.py b/emf/model_quality/model_quality.py index dcb266a2..18fefb76 100644 --- a/emf/model_quality/model_quality.py +++ b/emf/model_quality/model_quality.py @@ -28,8 +28,8 @@ def handle(self, message: bytes, properties: dict, **kwargs): object_type = properties.headers['opde:Object-Type'] if object_type == 'CGM': - model_data = self.minio_service.download_object(model_metadata.get('minio-bucket'), - model_metadata.get('content_reference')) + model_data = self.minio_service.download_object(model_metadata.get('minio-bucket', 'opde-confidential-models'), + model_metadata.get('pmd:content-reference')) logger.info(f"Loading merged model: {model_metadata['name']}") unzipped = process_zipped_cgm(model_data) network= load_all_to_dataframe(unzipped) @@ -53,7 +53,7 @@ def handle(self, message: bytes, properties: dict, **kwargs): # TODO move statistics function to quality functions file or move statistics file to quality directory model_statistics = get_system_metrics(network) except Exception as e: - model_statistics = None + model_statistics = {} logger.error(f"Failed to get model statistics: {e}") else: raise TypeError("Model was not loaded correctly, either missing in MinIO or incorrect data") diff --git a/emf/model_quality/model_statistics.py b/emf/model_quality/model_statistics.py index e5ef8958..952a431e 100644 --- a/emf/model_quality/model_statistics.py +++ b/emf/model_quality/model_statistics.py @@ -90,7 +90,11 @@ def get_tieflow_data(data): try: tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal->ConnectivityNode") except: - tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal-ConnectivityNode") + try: + tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal-ConnectivityNode") + except: + tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal-ConnectivityNode") + try: tieflow_data["BoundaryPoint.isDirectCurrent"] = tieflow_data["IdentifiedObject.description"].str.startswith("HVDC") diff --git a/emf/model_quality/quality_functions.py b/emf/model_quality/quality_functions.py index e5e4e49e..2bb20e38 100644 --- a/emf/model_quality/quality_functions.py +++ b/emf/model_quality/quality_functions.py @@ -8,7 +8,7 @@ def generate_quality_report(network, object_type, model_metadata): report = {} - if object_type == "CGM" and model_metadata['merge_type'] == 'BA': + if object_type == "CGM" and model_metadata['pmd:Area'] == 'BA': # Check Kruonis generators generators = network.type_tableview('SynchronousMachine').rename_axis('Terminal').reset_index() @@ -90,7 +90,11 @@ def generate_quality_report(network, object_type, model_metadata): elif object_type == "IGM": report = model_metadata[0] model_metadata[0].pop('opde:Component') - model_metadata[0].pop('opde:Dependencies') + try: + model_metadata[0].pop('opde:Dependencies') + except: + model_metadata[0].pop('opde:DependsOn') + report.update({"quality": "No Status"}) return report From 9be3577a094480aa11df1f4c3d2ccab45e8c8add Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Thu, 10 Jul 2025 11:21:19 +0300 Subject: [PATCH 63/67] bump to triplets 0.0.11 --- Pipfile.lock | 6 +++--- requirements.txt | Bin 3608 -> 3608 bytes 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 5ac6eea6..211aac17 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -698,11 +698,11 @@ }, "triplets": { "hashes": [ - "sha256:2ab66c9036a20a741cccbf1170fa446947ded88665736cab8f7f1c2a8d0f6235", - "sha256:edf08b458a7afc2001e0db71e0efa6198363bdeba6f4b07ef46fe99a292687f7" + "sha256:6f728262a185508a316c243e2efa2fb0478e961a564cd261938811d0ff09d8bb", + "sha256:eddfa949ae5a32e280595ecfaea576b63deb8427cda830f28080f76de16f6c70" ], "index": "pypi", - "version": "==0.0.10" + "version": "==0.0.11" }, "typing-extensions": { "hashes": [ diff --git a/requirements.txt b/requirements.txt index 7f22789bfd96f5b9dd0a036d2558ef632ddb77b6..a43f05b88d821d6fe67e0f66a285943fb229af31 100644 GIT binary patch delta 14 VcmbOsGec&B3lF2=W>=oei~u7|1aklY delta 14 VcmbOsGec&B3lF2gW>=oei~u7?1abfX From 58cb1bf9c1dce900533e26f556369b64cb77a495 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Thu, 10 Jul 2025 14:58:02 +0300 Subject: [PATCH 64/67] bugfix: replacement QUERY_FILTER env variable removal --- config/cgm_worker/replacement_conf.json | 10 +++++----- emf/model_merger/model_merger.py | 2 +- emf/model_merger/replacement.py | 13 ++++++++----- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/config/cgm_worker/replacement_conf.json b/config/cgm_worker/replacement_conf.json index 02e60153..cf83d639 100644 --- a/config/cgm_worker/replacement_conf.json +++ b/config/cgm_worker/replacement_conf.json @@ -2,13 +2,13 @@ "configuration": { "validity_start": "2024-02-22T00:00:00Z", "validity_end": "2024-02-22T23:59:59Z", - "timeZone": "CET" + "time_zone": "CET" }, - "timeHorizon": { - "Request_list": ["ID", "1D", "2D", "WK", "1D", "MO", "YR"] + "time_horizon": { + "request_list": ["ID", "1D", "2D", "WK", "1D", "MO", "YR"] }, - "replacementLength": { - "Request_list": ["2w", "2w", "2w", "2w", "2w", "1m", "4m"] + "replacement_length": { + "request_list": ["2w", "2w", "2w", "2w", "2w", "1m", "4m"] }, "month_ahead": { "hours": ["07:30", "08:30", "09:30", "06:30", "05:30", "04:30"], diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index 3cf834c7..0386bf0c 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -220,7 +220,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): else: if model_replacement: # Get TSOs who models are available in storage for replacement period - available_tsos = get_tsos_available_in_storage() + available_tsos = get_tsos_available_in_storage(time_horizon=time_horizon) valid_model_tsos = [model['pmd:TSO'] for model in models] # Need to ensure that excluded models by task configuration would not be taken in replacement context missing_models = [tso for tso in available_tsos if tso not in valid_model_tsos + excluded_models] diff --git a/emf/model_merger/replacement.py b/emf/model_merger/replacement.py index 3572bd69..0a6ead95 100644 --- a/emf/model_merger/replacement.py +++ b/emf/model_merger/replacement.py @@ -13,12 +13,13 @@ logger = logging.getLogger(__name__) parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.replacement) +replacement_config = json.load(config.paths.cgm_worker.replacement_conf) def run_replacement(tso_list: list, time_horizon: str, scenario_date: str, - config: list = json.load(config.paths.cgm_worker.replacement_conf), + config: list = replacement_config, data_source: str = 'OPDM', ): """ @@ -36,7 +37,7 @@ def run_replacement(tso_list: list, # TODO time horizon exclusion logic + exclude available models from query # TODO put in query object type if CGM metadata objects will be stored # Get replacement length by time horizon - query_filter = 'now-' + config["replacementLength"]["Request_list"][config["timeHorizon"]["Request_list"].index(time_horizon)] + query_filter = 'now-' + config["replacement_length"]["request_list"][config["time_horizon"]["request_list"].index(time_horizon)] # Query for available replacement models query = {"pmd:TSO.keyword": tso_list, "valid": True, "data-source": data_source} model_df = pd.DataFrame(query_data(query, query_filter)) @@ -156,7 +157,7 @@ def make_lists_priority(timestamp, target_timehorizon, conf): hour_list_final = list(map(lambda x: (date_time + parse_duration(x)).strftime("%H:%M"), hour_list)) day_list_final = list(map(lambda x: (date_time + parse_duration(x)).strftime("%Y-%m-%d"), day_list)) - business_list = conf["timeHorizon"]["Request_list"] + business_list = conf["time_horizon"]["request_list"] business_list_final = business_list[business_list.index(target_timehorizon):] # make list of relevant businesstypes # Month ahead requires separate replacement logic @@ -198,9 +199,11 @@ def create_replacement_table(target_timestamp, target_timehorizon, valid_models_ return valid_models_df -def get_tsos_available_in_storage(): +def get_tsos_available_in_storage(time_horizon: str): metadata = {"opde:Object-Type": "IGM", "valid": True} - unique_tsos = fetch_unique_values(metadata_query=metadata, field="pmd:TSO.keyword", query_filter=QUERY_FILTER) + # Get query length by time horizon from configuration + query_filter = 'now-' + replacement_config["replacement_length"]["request_list"][replacement_config["time_horizon"]["request_list"].index(time_horizon)] + unique_tsos = fetch_unique_values(metadata_query=metadata, field="pmd:TSO.keyword", query_filter=query_filter) return unique_tsos From 0a8583da680fe907d6dcb77f0d3c58a39c629a19 Mon Sep 17 00:00:00 2001 From: "martynas.karobcikas" Date: Fri, 11 Jul 2025 11:37:23 +0300 Subject: [PATCH 65/67] bugfix: task generator get task initiator does not work in kube --- emf/task_generator/task_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/emf/task_generator/task_generator.py b/emf/task_generator/task_generator.py index 52fbd313..b092bd06 100644 --- a/emf/task_generator/task_generator.py +++ b/emf/task_generator/task_generator.py @@ -5,7 +5,7 @@ import croniter import json import logging -from os import getlogin +import os from emf.common.helpers.time import parse_duration, convert_to_utc, convert_to_timezone, timezone, reference_times, utcnow from emf.common.helpers.tasks import update_task_status from emf.common.integrations.elastic import Elastic @@ -140,7 +140,7 @@ def generate_tasks(task_window_duration: str, "run_id": run.get("@id", None), "job_id": f"urn:uuid:{job_id}", "task_type": "automatic", - "task_initiator": getlogin(), + "task_initiator": os.environ.get("USERNAME", "unknown"), "task_priority": run.get("priority", process.get("priority", "normal")), # "low", "normal", "high" "task_creation_time": task_timestamp, "task_update_time": "", From 0a5b966ba9f6dec4d9f528f383df291525af943c Mon Sep 17 00:00:00 2001 From: Simon <69009411+snec-afk@users.noreply.github.com> Date: Fri, 11 Jul 2025 13:15:48 +0300 Subject: [PATCH 66/67] Update publish-workers.yml --- .github/workflows/publish-workers.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-workers.yml b/.github/workflows/publish-workers.yml index d559993e..a29ad167 100644 --- a/.github/workflows/publish-workers.yml +++ b/.github/workflows/publish-workers.yml @@ -5,9 +5,8 @@ on: push: tags: - "v*" - # branches: - # - main - workflow_dispatch: + branches: + - main jobs: build_workers: @@ -20,7 +19,6 @@ jobs: - emfos-model-validator - emfos-model-merger - emfos-task-generator - - emfos-model-quality steps: - uses: actions/checkout@v4 @@ -70,3 +68,10 @@ jobs: if [[ "$BRANCH_NAME" == *-dev ]]; then docker push "${IMAGE_NAME}:latest" fi + - name: Update helm values with new image tag + run: | + git config user.name "Github Actions" + git config user.email "actions@github.com" + git add helm-chart/test-rcc-cl2.values.yaml + git commit -m "Update image tag to ${{ env.IMAGE_TAG }} + git push From 98d4a9947cda16d72f36786a81927d8e8cc4ee4d Mon Sep 17 00:00:00 2001 From: Simon <69009411+snec-afk@users.noreply.github.com> Date: Fri, 11 Jul 2025 13:32:53 +0300 Subject: [PATCH 67/67] Update publish-workers.yml --- .github/workflows/publish-workers.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/publish-workers.yml b/.github/workflows/publish-workers.yml index a29ad167..68e8c389 100644 --- a/.github/workflows/publish-workers.yml +++ b/.github/workflows/publish-workers.yml @@ -68,10 +68,10 @@ jobs: if [[ "$BRANCH_NAME" == *-dev ]]; then docker push "${IMAGE_NAME}:latest" fi - - name: Update helm values with new image tag - run: | - git config user.name "Github Actions" - git config user.email "actions@github.com" - git add helm-chart/test-rcc-cl2.values.yaml - git commit -m "Update image tag to ${{ env.IMAGE_TAG }} - git push + # - name: Update helm values with new image tag + # run: | + # git config user.name "Github Actions" + # git config user.email "actions@github.com" + # git add helm-chart/test-rcc-cl2.values.yaml + # git commit -m "Update image tag to ${{ env.IMAGE_TAG }} + # git push