diff --git a/.github/workflows/publish-workers.yml b/.github/workflows/publish-workers.yml index ea2a7871..68e8c389 100644 --- a/.github/workflows/publish-workers.yml +++ b/.github/workflows/publish-workers.yml @@ -68,3 +68,10 @@ jobs: if [[ "$BRANCH_NAME" == *-dev ]]; then docker push "${IMAGE_NAME}:latest" fi + # - name: Update helm values with new image tag + # run: | + # git config user.name "Github Actions" + # git config user.email "actions@github.com" + # git add helm-chart/test-rcc-cl2.values.yaml + # git commit -m "Update image tag to ${{ env.IMAGE_TAG }} + # git push diff --git a/.python-version b/.python-version new file mode 100644 index 00000000..2c073331 --- /dev/null +++ b/.python-version @@ -0,0 +1 @@ +3.11 diff --git a/Pipfile b/Pipfile index 05391717..17e9c60e 100644 --- a/Pipfile +++ b/Pipfile @@ -6,7 +6,7 @@ name = "pypi" [packages] pandas = "2.2.2" numpy = "2.0.0" -pypowsybl = "1.9.0" +pypowsybl = "*" aniso8601 = "*" lxml = "*" edx = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 404d5b38..211aac17 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -18,11 +18,11 @@ "default": { "aniso8601": { "hashes": [ - "sha256:1d2b7ef82963909e93c4f24ce48d4de9e66009a21bf1c1e1c85bdd0812fe412f", - "sha256:72e3117667eedf66951bb2d93f4296a56b94b078a8a95905a052611fb3f1b973" + "sha256:25488f8663dd1528ae1f54f94ac1ea51ae25b4d531539b8bc707fed184d16845", + "sha256:eb19717fd4e0db6de1aab06f12450ab92144246b257423fe020af5748c0cb89e" ], "index": "pypi", - "version": "==9.0.1" + "version": "==10.0.1" }, "argon2-cffi": { "hashes": [ @@ -61,19 +61,19 @@ }, "attrs": { "hashes": [ - "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", - "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2" + "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", + "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b" ], - "markers": "python_version >= '3.7'", - "version": "==24.2.0" + "markers": "python_version >= '3.8'", + "version": "==25.3.0" }, "certifi": { "hashes": [ - "sha256:922820b53db7a7257ffbda3f597266d435245903d80737e34f8a45ff3e3230d8", - "sha256:bec941d2aa8195e248a60b31ff9f0558284cf01a52591ceda73ea9afffd69fd9" + "sha256:c1d2ec05395148ee10cf672ffc28cd37ea0ab0d99f9cc74c43e588cbd111b079", + "sha256:d842783a14f8fdd646895ac26f719a061408834473cfc10203f6a575beb15d39" ], - "markers": "python_version >= '3.6'", - "version": "==2024.8.30" + "markers": "python_version >= '3.7'", + "version": "==2025.7.9" }, "cffi": { "hashes": [ @@ -150,99 +150,101 @@ }, "charset-normalizer": { "hashes": [ - "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", - "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", - "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", - "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", - "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", - "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", - "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", - "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", - "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", - "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", - "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", - "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", - "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", - "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", - "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", - "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", - "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", - "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", - "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", - "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", - "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", - "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", - "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", - "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", - "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", - "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", - "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", - "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", - "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", - "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", - "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", - "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", - "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", - "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", - "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", - "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", - "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", - "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", - "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", - "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", - "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", - "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", - "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", - "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", - "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", - "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", - "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", - "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", - "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", - "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", - "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", - "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", - "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", - "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", - "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", - "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", - "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", - "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", - "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", - "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", - "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", - "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", - "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", - "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", - "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", - "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", - "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", - "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", - "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", - "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", - "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", - "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", - "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", - "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", - "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", - "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", - "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", - "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", - "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", - "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", - "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", - "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", - "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", - "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", - "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", - "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", - "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", - "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", - "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", - "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" + "sha256:005fa3432484527f9732ebd315da8da8001593e2cf46a3d817669f062c3d9ed4", + "sha256:046595208aae0120559a67693ecc65dd75d46f7bf687f159127046628178dc45", + "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7", + "sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0", + "sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7", + "sha256:18dd2e350387c87dabe711b86f83c9c78af772c748904d372ade190b5c7c9d4d", + "sha256:1b1bde144d98e446b056ef98e59c256e9294f6b74d7af6846bf5ffdafd687a7d", + "sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0", + "sha256:1cad5f45b3146325bb38d6855642f6fd609c3f7cad4dbaf75549bf3b904d3184", + "sha256:21b2899062867b0e1fde9b724f8aecb1af14f2778d69aacd1a5a1853a597a5db", + "sha256:24498ba8ed6c2e0b56d4acbf83f2d989720a93b41d712ebd4f4979660db4417b", + "sha256:25a23ea5c7edc53e0f29bae2c44fcb5a1aa10591aae107f2a2b2583a9c5cbc64", + "sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b", + "sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8", + "sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff", + "sha256:36b31da18b8890a76ec181c3cf44326bf2c48e36d393ca1b72b3f484113ea344", + "sha256:3c21d4fca343c805a52c0c78edc01e3477f6dd1ad7c47653241cf2a206d4fc58", + "sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e", + "sha256:43e0933a0eff183ee85833f341ec567c0980dae57c464d8a508e1b2ceb336471", + "sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148", + "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a", + "sha256:50bf98d5e563b83cc29471fa114366e6806bc06bc7a25fd59641e41445327836", + "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e", + "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", + "sha256:5bf4545e3b962767e5c06fe1738f951f77d27967cb2caa64c28be7c4563e162c", + "sha256:6333b3aa5a12c26b2a4d4e7335a28f1475e0e5e17d69d55141ee3cab736f66d1", + "sha256:65c981bdbd3f57670af8b59777cbfae75364b483fa8a9f420f08094531d54a01", + "sha256:68a328e5f55ec37c57f19ebb1fdc56a248db2e3e9ad769919a58672958e8f366", + "sha256:6a0289e4589e8bdfef02a80478f1dfcb14f0ab696b5a00e1f4b8a14a307a3c58", + "sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5", + "sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c", + "sha256:6fc1f5b51fa4cecaa18f2bd7a003f3dd039dd615cd69a2afd6d3b19aed6775f2", + "sha256:70f7172939fdf8790425ba31915bfbe8335030f05b9913d7ae00a87d4395620a", + "sha256:721c76e84fe669be19c5791da68232ca2e05ba5185575086e384352e2c309597", + "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b", + "sha256:75d10d37a47afee94919c4fab4c22b9bc2a8bf7d4f46f87363bcf0573f3ff4f5", + "sha256:76af085e67e56c8816c3ccf256ebd136def2ed9654525348cfa744b6802b69eb", + "sha256:770cab594ecf99ae64c236bc9ee3439c3f46be49796e265ce0cc8bc17b10294f", + "sha256:7a6ab32f7210554a96cd9e33abe3ddd86732beeafc7a28e9955cdf22ffadbab0", + "sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941", + "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", + "sha256:8075c35cd58273fee266c58c0c9b670947c19df5fb98e7b66710e04ad4e9ff86", + "sha256:8272b73e1c5603666618805fe821edba66892e2870058c94c53147602eab29c7", + "sha256:82d8fd25b7f4675d0c47cf95b594d4e7b158aca33b76aa63d07186e13c0e0ab7", + "sha256:844da2b5728b5ce0e32d863af26f32b5ce61bc4273a9c720a9f3aa9df73b1455", + "sha256:8755483f3c00d6c9a77f490c17e6ab0c8729e39e6390328e42521ef175380ae6", + "sha256:915f3849a011c1f593ab99092f3cecfcb4d65d8feb4a64cf1bf2d22074dc0ec4", + "sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0", + "sha256:982bb1e8b4ffda883b3d0a521e23abcd6fd17418f6d2c4118d257a10199c0ce3", + "sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1", + "sha256:9cbfacf36cb0ec2897ce0ebc5d08ca44213af24265bd56eca54bee7923c48fd6", + "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981", + "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c", + "sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980", + "sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645", + "sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7", + "sha256:aaf27faa992bfee0264dc1f03f4c75e9fcdda66a519db6b957a3f826e285cf12", + "sha256:b2680962a4848b3c4f155dc2ee64505a9c57186d0d56b43123b17ca3de18f0fa", + "sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd", + "sha256:b33de11b92e9f75a2b545d6e9b6f37e398d86c3e9e9653c4864eb7e89c5773ef", + "sha256:b3daeac64d5b371dea99714f08ffc2c208522ec6b06fbc7866a450dd446f5c0f", + "sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2", + "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d", + "sha256:c72fbbe68c6f32f251bdc08b8611c7b3060612236e960ef848e0a517ddbe76c5", + "sha256:c9e36a97bee9b86ef9a1cf7bb96747eb7a15c2f22bdb5b516434b00f2a599f02", + "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3", + "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd", + "sha256:d11b54acf878eef558599658b0ffca78138c8c3655cf4f3a4a673c437e67732e", + "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214", + "sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd", + "sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a", + "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c", + "sha256:dc7039885fa1baf9be153a0626e337aa7ec8bf96b0128605fb0d77788ddc1681", + "sha256:dccab8d5fa1ef9bfba0590ecf4d46df048d18ffe3eec01eeb73a42e0d9e7a8ba", + "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f", + "sha256:e45ba65510e2647721e35323d6ef54c7974959f6081b58d4ef5d87c60c84919a", + "sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28", + "sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691", + "sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82", + "sha256:e8082b26888e2f8b36a042a58307d5b917ef2b1cacab921ad3323ef91901c71a", + "sha256:e8323a9b031aa0393768b87f04b4164a40037fb2a3c11ac06a03ffecd3618027", + "sha256:e92fca20c46e9f5e1bb485887d074918b13543b1c2a1185e69bb8d17ab6236a7", + "sha256:eb30abc20df9ab0814b5a2524f23d75dcf83cde762c161917a2b4b7b55b1e518", + "sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf", + "sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b", + "sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9", + "sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544", + "sha256:f4074c5a429281bf056ddd4c5d3b740ebca4d43ffffe2ef4bf4d2d05114299da", + "sha256:f69a27e45c43520f5487f27627059b64aaf160415589230992cec34c5e18a509", + "sha256:fb707f3e15060adf5b7ada797624a6c6e0138e2a26baa089df64c68ee98e040f", + "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a", + "sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f" ], - "markers": "python_full_version >= '3.7.0'", - "version": "==3.3.2" + "markers": "python_version >= '3.7'", + "version": "==3.4.2" }, "croniter": { "hashes": [ @@ -280,163 +282,120 @@ }, "idna": { "hashes": [ - "sha256:050b4e5baadcd44d760cedbd2b8e639f2ff89bbc7a5730fcc662954303377aac", - "sha256:d838c2c0ed6fced7693d5e8ab8e734d5f8fda53a039c0164afb0b82e771e3603" + "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", + "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3" ], "markers": "python_version >= '3.6'", - "version": "==3.8" + "version": "==3.10" }, "isodate": { "hashes": [ - "sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96", - "sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9" + "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", + "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6" ], - "version": "==0.6.1" + "markers": "python_version >= '3.7'", + "version": "==0.7.2" }, "lxml": { "hashes": [ - "sha256:01220dca0d066d1349bd6a1726856a78f7929f3878f7e2ee83c296c69495309e", - "sha256:02ced472497b8362c8e902ade23e3300479f4f43e45f4105c85ef43b8db85229", - "sha256:052d99051e77a4f3e8482c65014cf6372e61b0a6f4fe9edb98503bb5364cfee3", - "sha256:07da23d7ee08577760f0a71d67a861019103e4812c87e2fab26b039054594cc5", - "sha256:094cb601ba9f55296774c2d57ad68730daa0b13dc260e1f941b4d13678239e70", - "sha256:0a7056921edbdd7560746f4221dca89bb7a3fe457d3d74267995253f46343f15", - "sha256:0c120f43553ec759f8de1fee2f4794452b0946773299d44c36bfe18e83caf002", - "sha256:0d7b36afa46c97875303a94e8f3ad932bf78bace9e18e603f2085b652422edcd", - "sha256:0fdf3a3059611f7585a78ee10399a15566356116a4288380921a4b598d807a22", - "sha256:109fa6fede314cc50eed29e6e56c540075e63d922455346f11e4d7a036d2b8cf", - "sha256:146173654d79eb1fc97498b4280c1d3e1e5d58c398fa530905c9ea50ea849b22", - "sha256:1473427aff3d66a3fa2199004c3e601e6c4500ab86696edffdbc84954c72d832", - "sha256:1483fd3358963cc5c1c9b122c80606a3a79ee0875bcac0204149fa09d6ff2727", - "sha256:168f2dfcfdedf611eb285efac1516c8454c8c99caf271dccda8943576b67552e", - "sha256:17e8d968d04a37c50ad9c456a286b525d78c4a1c15dd53aa46c1d8e06bf6fa30", - "sha256:18feb4b93302091b1541221196a2155aa296c363fd233814fa11e181adebc52f", - "sha256:1afe0a8c353746e610bd9031a630a95bcfb1a720684c3f2b36c4710a0a96528f", - "sha256:1d04f064bebdfef9240478f7a779e8c5dc32b8b7b0b2fc6a62e39b928d428e51", - "sha256:1fdc9fae8dd4c763e8a31e7630afef517eab9f5d5d31a278df087f307bf601f4", - "sha256:1ffc23010330c2ab67fac02781df60998ca8fe759e8efde6f8b756a20599c5de", - "sha256:20094fc3f21ea0a8669dc4c61ed7fa8263bd37d97d93b90f28fc613371e7a875", - "sha256:213261f168c5e1d9b7535a67e68b1f59f92398dd17a56d934550837143f79c42", - "sha256:218c1b2e17a710e363855594230f44060e2025b05c80d1f0661258142b2add2e", - "sha256:23e0553b8055600b3bf4a00b255ec5c92e1e4aebf8c2c09334f8368e8bd174d6", - "sha256:25f1b69d41656b05885aa185f5fdf822cb01a586d1b32739633679699f220391", - "sha256:2b3778cb38212f52fac9fe913017deea2fdf4eb1a4f8e4cfc6b009a13a6d3fcc", - "sha256:2bc9fd5ca4729af796f9f59cd8ff160fe06a474da40aca03fcc79655ddee1a8b", - "sha256:2c226a06ecb8cdef28845ae976da407917542c5e6e75dcac7cc33eb04aaeb237", - "sha256:2c3406b63232fc7e9b8783ab0b765d7c59e7c59ff96759d8ef9632fca27c7ee4", - "sha256:2c86bf781b12ba417f64f3422cfc302523ac9cd1d8ae8c0f92a1c66e56ef2e86", - "sha256:2d9b8d9177afaef80c53c0a9e30fa252ff3036fb1c6494d427c066a4ce6a282f", - "sha256:2dec2d1130a9cda5b904696cec33b2cfb451304ba9081eeda7f90f724097300a", - "sha256:2dfab5fa6a28a0b60a20638dc48e6343c02ea9933e3279ccb132f555a62323d8", - "sha256:2ecdd78ab768f844c7a1d4a03595038c166b609f6395e25af9b0f3f26ae1230f", - "sha256:315f9542011b2c4e1d280e4a20ddcca1761993dda3afc7a73b01235f8641e903", - "sha256:36aef61a1678cb778097b4a6eeae96a69875d51d1e8f4d4b491ab3cfb54b5a03", - "sha256:384aacddf2e5813a36495233b64cb96b1949da72bef933918ba5c84e06af8f0e", - "sha256:3879cc6ce938ff4eb4900d901ed63555c778731a96365e53fadb36437a131a99", - "sha256:3c174dc350d3ec52deb77f2faf05c439331d6ed5e702fc247ccb4e6b62d884b7", - "sha256:3eb44520c4724c2e1a57c0af33a379eee41792595023f367ba3952a2d96c2aab", - "sha256:406246b96d552e0503e17a1006fd27edac678b3fcc9f1be71a2f94b4ff61528d", - "sha256:41ce1f1e2c7755abfc7e759dc34d7d05fd221723ff822947132dc934d122fe22", - "sha256:423b121f7e6fa514ba0c7918e56955a1d4470ed35faa03e3d9f0e3baa4c7e492", - "sha256:44264ecae91b30e5633013fb66f6ddd05c006d3e0e884f75ce0b4755b3e3847b", - "sha256:482c2f67761868f0108b1743098640fbb2a28a8e15bf3f47ada9fa59d9fe08c3", - "sha256:4b0c7a688944891086ba192e21c5229dea54382f4836a209ff8d0a660fac06be", - "sha256:4c1fefd7e3d00921c44dc9ca80a775af49698bbfd92ea84498e56acffd4c5469", - "sha256:4e109ca30d1edec1ac60cdbe341905dc3b8f55b16855e03a54aaf59e51ec8c6f", - "sha256:501d0d7e26b4d261fca8132854d845e4988097611ba2531408ec91cf3fd9d20a", - "sha256:516f491c834eb320d6c843156440fe7fc0d50b33e44387fcec5b02f0bc118a4c", - "sha256:51806cfe0279e06ed8500ce19479d757db42a30fd509940b1701be9c86a5ff9a", - "sha256:562e7494778a69086f0312ec9689f6b6ac1c6b65670ed7d0267e49f57ffa08c4", - "sha256:56b9861a71575f5795bde89256e7467ece3d339c9b43141dbdd54544566b3b94", - "sha256:5b8f5db71b28b8c404956ddf79575ea77aa8b1538e8b2ef9ec877945b3f46442", - "sha256:5c2fb570d7823c2bbaf8b419ba6e5662137f8166e364a8b2b91051a1fb40ab8b", - "sha256:5c54afdcbb0182d06836cc3d1be921e540be3ebdf8b8a51ee3ef987537455f84", - "sha256:5d6a6972b93c426ace71e0be9a6f4b2cfae9b1baed2eed2006076a746692288c", - "sha256:609251a0ca4770e5a8768ff902aa02bf636339c5a93f9349b48eb1f606f7f3e9", - "sha256:62d172f358f33a26d6b41b28c170c63886742f5b6772a42b59b4f0fa10526cb1", - "sha256:62f7fdb0d1ed2065451f086519865b4c90aa19aed51081979ecd05a21eb4d1be", - "sha256:658f2aa69d31e09699705949b5fc4719cbecbd4a97f9656a232e7d6c7be1a367", - "sha256:65ab5685d56914b9a2a34d67dd5488b83213d680b0c5d10b47f81da5a16b0b0e", - "sha256:68934b242c51eb02907c5b81d138cb977b2129a0a75a8f8b60b01cb8586c7b21", - "sha256:68b87753c784d6acb8a25b05cb526c3406913c9d988d51f80adecc2b0775d6aa", - "sha256:69959bd3167b993e6e710b99051265654133a98f20cec1d9b493b931942e9c16", - "sha256:6a7095eeec6f89111d03dabfe5883a1fd54da319c94e0fb104ee8f23616b572d", - "sha256:6b038cc86b285e4f9fea2ba5ee76e89f21ed1ea898e287dc277a25884f3a7dfe", - "sha256:6ba0d3dcac281aad8a0e5b14c7ed6f9fa89c8612b47939fc94f80b16e2e9bc83", - "sha256:6e91cf736959057f7aac7adfc83481e03615a8e8dd5758aa1d95ea69e8931dba", - "sha256:6ee8c39582d2652dcd516d1b879451500f8db3fe3607ce45d7c5957ab2596040", - "sha256:6f651ebd0b21ec65dfca93aa629610a0dbc13dbc13554f19b0113da2e61a4763", - "sha256:71a8dd38fbd2f2319136d4ae855a7078c69c9a38ae06e0c17c73fd70fc6caad8", - "sha256:74068c601baff6ff021c70f0935b0c7bc528baa8ea210c202e03757c68c5a4ff", - "sha256:7437237c6a66b7ca341e868cda48be24b8701862757426852c9b3186de1da8a2", - "sha256:747a3d3e98e24597981ca0be0fd922aebd471fa99d0043a3842d00cdcad7ad6a", - "sha256:74bcb423462233bc5d6066e4e98b0264e7c1bed7541fff2f4e34fe6b21563c8b", - "sha256:78d9b952e07aed35fe2e1a7ad26e929595412db48535921c5013edc8aa4a35ce", - "sha256:7b1cd427cb0d5f7393c31b7496419da594fe600e6fdc4b105a54f82405e6626c", - "sha256:7d3d1ca42870cdb6d0d29939630dbe48fa511c203724820fc0fd507b2fb46577", - "sha256:7e2f58095acc211eb9d8b5771bf04df9ff37d6b87618d1cbf85f92399c98dae8", - "sha256:7f41026c1d64043a36fda21d64c5026762d53a77043e73e94b71f0521939cc71", - "sha256:81b4e48da4c69313192d8c8d4311e5d818b8be1afe68ee20f6385d0e96fc9512", - "sha256:86a6b24b19eaebc448dc56b87c4865527855145d851f9fc3891673ff97950540", - "sha256:874a216bf6afaf97c263b56371434e47e2c652d215788396f60477540298218f", - "sha256:89e043f1d9d341c52bf2af6d02e6adde62e0a46e6755d5eb60dc6e4f0b8aeca2", - "sha256:8c72e9563347c7395910de6a3100a4840a75a6f60e05af5e58566868d5eb2d6a", - "sha256:8dc2c0395bea8254d8daebc76dcf8eb3a95ec2a46fa6fae5eaccee366bfe02ce", - "sha256:8f0de2d390af441fe8b2c12626d103540b5d850d585b18fcada58d972b74a74e", - "sha256:92e67a0be1639c251d21e35fe74df6bcc40cba445c2cda7c4a967656733249e2", - "sha256:94d6c3782907b5e40e21cadf94b13b0842ac421192f26b84c45f13f3c9d5dc27", - "sha256:97acf1e1fd66ab53dacd2c35b319d7e548380c2e9e8c54525c6e76d21b1ae3b1", - "sha256:9ada35dd21dc6c039259596b358caab6b13f4db4d4a7f8665764d616daf9cc1d", - "sha256:9c52100e2c2dbb0649b90467935c4b0de5528833c76a35ea1a2691ec9f1ee7a1", - "sha256:9e41506fec7a7f9405b14aa2d5c8abbb4dbbd09d88f9496958b6d00cb4d45330", - "sha256:9e4b47ac0f5e749cfc618efdf4726269441014ae1d5583e047b452a32e221920", - "sha256:9fb81d2824dff4f2e297a276297e9031f46d2682cafc484f49de182aa5e5df99", - "sha256:a0eabd0a81625049c5df745209dc7fcef6e2aea7793e5f003ba363610aa0a3ff", - "sha256:a3d819eb6f9b8677f57f9664265d0a10dd6551d227afb4af2b9cd7bdc2ccbf18", - "sha256:a87de7dd873bf9a792bf1e58b1c3887b9264036629a5bf2d2e6579fe8e73edff", - "sha256:aa617107a410245b8660028a7483b68e7914304a6d4882b5ff3d2d3eb5948d8c", - "sha256:aac0bbd3e8dd2d9c45ceb82249e8bdd3ac99131a32b4d35c8af3cc9db1657179", - "sha256:ab6dd83b970dc97c2d10bc71aa925b84788c7c05de30241b9e96f9b6d9ea3080", - "sha256:ace2c2326a319a0bb8a8b0e5b570c764962e95818de9f259ce814ee666603f19", - "sha256:ae5fe5c4b525aa82b8076c1a59d642c17b6e8739ecf852522c6321852178119d", - "sha256:b11a5d918a6216e521c715b02749240fb07ae5a1fefd4b7bf12f833bc8b4fe70", - "sha256:b1c8c20847b9f34e98080da785bb2336ea982e7f913eed5809e5a3c872900f32", - "sha256:b369d3db3c22ed14c75ccd5af429086f166a19627e84a8fdade3f8f31426e52a", - "sha256:b710bc2b8292966b23a6a0121f7a6c51d45d2347edcc75f016ac123b8054d3f2", - "sha256:bd96517ef76c8654446fc3db9242d019a1bb5fe8b751ba414765d59f99210b79", - "sha256:c00f323cc00576df6165cc9d21a4c21285fa6b9989c5c39830c3903dc4303ef3", - "sha256:c162b216070f280fa7da844531169be0baf9ccb17263cf5a8bf876fcd3117fa5", - "sha256:c1a69e58a6bb2de65902051d57fde951febad631a20a64572677a1052690482f", - "sha256:c1f794c02903c2824fccce5b20c339a1a14b114e83b306ff11b597c5f71a1c8d", - "sha256:c24037349665434f375645fa9d1f5304800cec574d0310f618490c871fd902b3", - "sha256:c300306673aa0f3ed5ed9372b21867690a17dba38c68c44b287437c362ce486b", - "sha256:c56a1d43b2f9ee4786e4658c7903f05da35b923fb53c11025712562d5cc02753", - "sha256:c6379f35350b655fd817cd0d6cbeef7f265f3ae5fedb1caae2eb442bbeae9ab9", - "sha256:c802e1c2ed9f0c06a65bc4ed0189d000ada8049312cfeab6ca635e39c9608957", - "sha256:cb83f8a875b3d9b458cada4f880fa498646874ba4011dc974e071a0a84a1b033", - "sha256:cf120cce539453ae086eacc0130a324e7026113510efa83ab42ef3fcfccac7fb", - "sha256:dd36439be765e2dde7660212b5275641edbc813e7b24668831a5c8ac91180656", - "sha256:dd5350b55f9fecddc51385463a4f67a5da829bc741e38cf689f38ec9023f54ab", - "sha256:df5c7333167b9674aa8ae1d4008fa4bc17a313cc490b2cca27838bbdcc6bb15b", - "sha256:e63601ad5cd8f860aa99d109889b5ac34de571c7ee902d6812d5d9ddcc77fa7d", - "sha256:e92ce66cd919d18d14b3856906a61d3f6b6a8500e0794142338da644260595cd", - "sha256:e99f5507401436fdcc85036a2e7dc2e28d962550afe1cbfc07c40e454256a859", - "sha256:ea2e2f6f801696ad7de8aec061044d6c8c0dd4037608c7cab38a9a4d316bfb11", - "sha256:eafa2c8658f4e560b098fe9fc54539f86528651f61849b22111a9b107d18910c", - "sha256:ecd4ad8453ac17bc7ba3868371bffb46f628161ad0eefbd0a855d2c8c32dd81a", - "sha256:ee70d08fd60c9565ba8190f41a46a54096afa0eeb8f76bd66f2c25d3b1b83005", - "sha256:eec1bb8cdbba2925bedc887bc0609a80e599c75b12d87ae42ac23fd199445654", - "sha256:ef0c1fe22171dd7c7c27147f2e9c3e86f8bdf473fed75f16b0c2e84a5030ce80", - "sha256:f2901429da1e645ce548bf9171784c0f74f0718c3f6150ce166be39e4dd66c3e", - "sha256:f422a209d2455c56849442ae42f25dbaaba1c6c3f501d58761c619c7836642ec", - "sha256:f65e5120863c2b266dbcc927b306c5b78e502c71edf3295dfcb9501ec96e5fc7", - "sha256:f7d4a670107d75dfe5ad080bed6c341d18c4442f9378c9f58e5851e86eb79965", - "sha256:f914c03e6a31deb632e2daa881fe198461f4d06e57ac3d0e05bbcab8eae01945", - "sha256:fb66442c2546446944437df74379e9cf9e9db353e61301d1a0e26482f43f0dd8" + "sha256:013090383863b72c62a702d07678b658fa2567aa58d373d963cca245b017e065", + "sha256:032e65120339d44cdc3efc326c9f660f5f7205f3a535c1fdbf898b29ea01fb72", + "sha256:048a930eb4572829604982e39a0c7289ab5dc8abc7fc9f5aabd6fbc08c154e93", + "sha256:04d67ceee6db4bcb92987ccb16e53bef6b42ced872509f333c04fb58a3315256", + "sha256:059c4cbf3973a621b62ea3132934ae737da2c132a788e6cfb9b08d63a0ef73f9", + "sha256:0e32698462aacc5c1cf6bdfebc9c781821b7e74c79f13e5ffc8bfe27c42b1abf", + "sha256:1676b56d48048a62ef77a250428d1f31f610763636e0784ba67a9740823988ca", + "sha256:17f090a9bc0ce8da51a5632092f98a7e7f84bca26f33d161a98b57f7fb0004ca", + "sha256:185efc2fed89cdd97552585c624d3c908f0464090f4b91f7d92f8ed2f3b18f54", + "sha256:1fa377b827ca2023244a06554c6e7dc6828a10aaf74ca41965c5d8a4925aebb4", + "sha256:2181e4b1d07dde53986023482673c0f1fba5178ef800f9ab95ad791e8bdded6a", + "sha256:219e0431ea8006e15005767f0351e3f7f9143e793e58519dc97fe9e07fae5563", + "sha256:21db1ec5525780fd07251636eb5f7acb84003e9382c72c18c542a87c416ade03", + "sha256:246b40f8a4aec341cbbf52617cad8ab7c888d944bfe12a6abd2b1f6cfb6f6082", + "sha256:2793a627e95d119e9f1e19720730472f5543a6d84c50ea33313ce328d870f2dd", + "sha256:2930aa001a3776c3e2601cb8e0a15d21b8270528d89cc308be4843ade546b9ab", + "sha256:2ae06fbab4f1bb7db4f7c8ca9897dc8db4447d1a2b9bee78474ad403437bcc29", + "sha256:2b4790b558bee331a933e08883c423f65bbcd07e278f91b2272489e31ab1e2b4", + "sha256:2cfcf84f1defed7e5798ef4f88aa25fcc52d279be731ce904789aa7ccfb7e8d2", + "sha256:2dd1cc3ea7e60bfb31ff32cafe07e24839df573a5e7c2d33304082a5019bcd58", + "sha256:2f34687222b78fff795feeb799a7d44eca2477c3d9d3a46ce17d51a4f383e32e", + "sha256:310b719b695b3dd442cdfbbe64936b2f2e231bb91d998e99e6f0daf991a3eba3", + "sha256:34190a1ec4f1e84af256495436b2d196529c3f2094f0af80202947567fdbf2e7", + "sha256:35bc626eec405f745199200ccb5c6b36f202675d204aa29bb52e27ba2b71dea8", + "sha256:36531f81c8214e293097cd2b7873f178997dae33d3667caaae8bdfb9666b76c0", + "sha256:390240baeb9f415a82eefc2e13285016f9c8b5ad71ec80574ae8fa9605093cd7", + "sha256:40442e2a4456e9910875ac12951476d36c0870dcb38a68719f8c4686609897c4", + "sha256:4337e4aec93b7c011f7ee2e357b0d30562edd1955620fdd4aeab6aacd90d43c5", + "sha256:43cfbb7db02b30ad3926e8fceaef260ba2fb7df787e38fa2df890c1ca7966c3b", + "sha256:43fe5af2d590bf4691531b1d9a2495d7aab2090547eaacd224a3afec95706d76", + "sha256:46b9ed911f36bfeb6338e0b482e7fe7c27d362c52fde29f221fddbc9ee2227e7", + "sha256:4d23854ecf381ab1facc8f353dcd9adeddef3652268ee75297c1164c987c11dc", + "sha256:4d6036c3a296707357efb375cfc24bb64cd955b9ec731abf11ebb1e40063949f", + "sha256:4eb114a0754fd00075c12648d991ec7a4357f9cb873042cc9a77bf3a7e30c9db", + "sha256:4ee56288d0df919e4aac43b539dd0e34bb55d6a12a6562038e8d6f3ed07f9e36", + "sha256:51a5e4c61a4541bd1cd3ba74766d0c9b6c12d6a1a4964ef60026832aac8e79b3", + "sha256:522fe7abb41309e9543b0d9b8b434f2b630c5fdaf6482bee642b34c8c70079c8", + "sha256:54c4855eabd9fc29707d30141be99e5cd1102e7d2258d2892314cf4c110726c3", + "sha256:5592401cdf3dc682194727c1ddaa8aa0f3ddc57ca64fd03226a430b955eab6f6", + "sha256:58ffd35bd5425c3c3b9692d078bf7ab851441434531a7e517c4984d5634cd65b", + "sha256:5967fe415b1920a3877a4195e9a2b779249630ee49ece22021c690320ff07452", + "sha256:5fcd7d3b1d8ecb91445bd71b9c88bdbeae528fefee4f379895becfc72298d181", + "sha256:63b634facdfbad421d4b61c90735688465d4ab3a8853ac22c76ccac2baf98d97", + "sha256:690b20e3388a7ec98e899fd54c924e50ba6693874aa65ef9cb53de7f7de9d64a", + "sha256:6da7cd4f405fd7db56e51e96bff0865b9853ae70df0e6720624049da76bde2da", + "sha256:7488a43033c958637b1a08cddc9188eb06d3ad36582cebc7d4815980b47e27ef", + "sha256:74e748012f8c19b47f7d6321ac929a9a94ee92ef12bc4298c47e8b7219b26541", + "sha256:78718d8454a6e928470d511bf8ac93f469283a45c354995f7d19e77292f26108", + "sha256:7bf61bc4345c1895221357af8f3e89f8c103d93156ef326532d35c707e2fb19d", + "sha256:7da298e1659e45d151b4028ad5c7974917e108afb48731f4ed785d02b6818994", + "sha256:84ef591495ffd3f9dcabffd6391db7bb70d7230b5c35ef5148354a134f56f2be", + "sha256:85b14a4689d5cff426c12eefe750738648706ea2753b20c2f973b2a000d3d261", + "sha256:8a2e76efbf8772add72d002d67a4c3d0958638696f541734304c7f28217a9cab", + "sha256:8a78d6c9168f5bcb20971bf3329c2b83078611fbe1f807baadc64afc70523b3a", + "sha256:8cb26f51c82d77483cdcd2b4a53cda55bbee29b3c2f3ddeb47182a2a9064e4eb", + "sha256:8db5dc617cb937ae17ff3403c3a70a7de9df4852a046f93e71edaec678f721d0", + "sha256:9ab542c91f5a47aaa58abdd8ea84b498e8e49fe4b883d67800017757a3eb78e8", + "sha256:9da022c14baeec36edfcc8daf0e281e2f55b950249a455776f0d1adeeada4734", + "sha256:9f4b481b6cc3a897adb4279216695150bbe7a44c03daba3c894f49d2037e0a24", + "sha256:a52a4704811e2623b0324a18d41ad4b9fabf43ce5ff99b14e40a520e2190c851", + "sha256:a55da151d0b0c6ab176b4e761670ac0e2667817a1e0dadd04a01d0561a219349", + "sha256:a674c0948789e9136d69065cc28009c1b1874c6ea340253db58be7622ce6398f", + "sha256:ae74f7c762270196d2dda56f8dd7309411f08a4084ff2dfcc0b095a218df2e06", + "sha256:afd27d8629ae94c5d863e32ab0e1d5590371d296b87dae0a751fb22bf3685741", + "sha256:b2d71cdefda9424adff9a3607ba5bbfc60ee972d73c21c7e3c19e71037574816", + "sha256:b34339898bb556a2351a1830f88f751679f343eabf9cf05841c95b165152c9e7", + "sha256:b372d10d17a701b0945f67be58fae4664fd056b85e0ff0fbc1e6c951cdbc0512", + "sha256:b3c98d5b24c6095e89e03d65d5c574705be3d49c0d8ca10c17a8a4b5201b72f5", + "sha256:b8dd6dd0e9c1992613ccda2bcb74fc9d49159dbe0f0ca4753f37527749885c25", + "sha256:bd5913b4972681ffc9718bc2d4c53cde39ef81415e1671ff93e9aa30b46595e7", + "sha256:c0b5fa5eda84057a4f1bbb4bb77a8c28ff20ae7ce211588d698ae453e13c6281", + "sha256:c16304bba98f48a28ae10e32a8e75c349dd742c45156f297e16eeb1ba9287a1f", + "sha256:c24b8efd9c0f62bad0439283c2c795ef916c5a6b75f03c17799775c7ae3c0c9e", + "sha256:c2a5e8d207311a0170aca0eb6b160af91adc29ec121832e4ac151a57743a1e1e", + "sha256:c352fc8f36f7e9727db17adbf93f82499457b3d7e5511368569b4c5bd155a922", + "sha256:c86df1c9af35d903d2b52d22ea3e66db8058d21dc0f59842ca5deb0595921141", + "sha256:c907516d49f77f6cd8ead1322198bdfd902003c3c330c77a1c5f3cc32a0e4d16", + "sha256:ca50bd612438258a91b5b3788c6621c1f05c8c478e7951899f492be42defc0da", + "sha256:d18a25b19ca7307045581b18b3ec9ead2b1db5ccd8719c291f0cd0a5cec6cb81", + "sha256:d4f0c66df4386b75d2ab1e20a489f30dc7fd9a06a896d64980541506086be1f1", + "sha256:d6e200909a119626744dd81bae409fc44134389e03fbf1d68ed2a55a2fb10991", + "sha256:d7ae472f74afcc47320238b5dbfd363aba111a525943c8a34a1b657c6be934c3", + "sha256:db0efd6bae1c4730b9c863fc4f5f3c0fa3e8f05cae2c44ae141cb9dfc7d091dc", + "sha256:dbdd7679a6f4f08152818043dbb39491d1af3332128b3752c3ec5cebc0011a72", + "sha256:e0b1520ef900e9ef62e392dd3d7ae4f5fa224d1dd62897a792cf353eb20b6cae", + "sha256:e2030956cf4886b10be9a0285c6802e078ec2391e1dd7ff3eb509c2c95a69b76", + "sha256:e35e8aaaf3981489f42884b59726693de32dabfc438ac10ef4eb3409961fd402", + "sha256:e380e85b93f148ad28ac15f8117e2fd8e5437aa7732d65e260134f83ce67911b", + "sha256:edf6e4c8fe14dfe316939711e3ece3f9a20760aabf686051b537a7562f4da91a", + "sha256:f3389924581d9a770c6caa4df4e74b606180869043b9073e2cec324bad6e306e", + "sha256:f64ccf593916e93b8d36ed55401bb7fe9c7d5de3180ce2e10b08f82a8f397316", + "sha256:f720a14aa102a38907c6d5030e3d66b3b680c3e6f6bc95473931ea3c00c59967", + "sha256:f8d19565ae3eb956d84da3ef367aa7def14a2735d05bd275cd54c0301f0d0d6c", + "sha256:f97487996a39cb18278ca33f7be98198f278d0bc3c5d0fd4d7b3d63646ca3c8a" ], "index": "pypi", - "markers": "python_version >= '3.6'", - "version": "==5.3.0" + "markers": "python_version >= '3.8'", + "version": "==6.0.0" }, "minio": { "hashes": [ @@ -457,11 +416,11 @@ }, "networkx": { "hashes": [ - "sha256:307c3669428c5362aab27c8a1260aa8f47c4e91d3891f48be0141738d8d053e1", - "sha256:df5d4365b724cf81b8c6a7312509d0c22386097011ad1abe274afd5e9d3bbc5f" + "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", + "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037" ], - "markers": "python_version >= '3.10'", - "version": "==3.4.2" + "markers": "python_version >= '3.11'", + "version": "==3.5" }, "numpy": { "hashes": [ @@ -517,11 +476,11 @@ }, "opdm-api": { "hashes": [ - "sha256:5191a42a228bdfeb5025dfd8c63d59128e3e637990952a002a79ddb5aae4c04e", - "sha256:7d95536756d5fb09dc17cceed3ad1b2db95aaa04a6bf3ed5ee3e168b4600e4ce" + "sha256:91913afc4a74e5fdeea5ba4c3442382342e0031b6384dbb05568691ff9495ef2", + "sha256:e5e1884d848190232ec62b7467e80451286a0c3c845faa87fbbf68837cc12c35" ], "index": "pypi", - "version": "==0.1.0" + "version": "==0.1.3" }, "pandas": { "hashes": [ @@ -570,19 +529,19 @@ }, "platformdirs": { "hashes": [ - "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee", - "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3" + "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc", + "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4" ], - "markers": "python_version >= '3.8'", - "version": "==4.2.2" + "markers": "python_version >= '3.9'", + "version": "==4.3.8" }, "prettytable": { "hashes": [ - "sha256:61d5c68f04a94acc73c7aac64f0f380f5bed4d2959d59edc6e4cbb7a0e7b55c4", - "sha256:b804b8d51db23959b96b329094debdbbdf10c8c3aa75958c5988cfd7f78501dd" + "sha256:3c64b31719d961bf69c9a7e03d0c1e477320906a98da63952bc6698d6164ff57", + "sha256:b5eccfabb82222f5aa46b798ff02a8452cf530a352c31bddfa29be41242863aa" ], "markers": "python_version >= '3.9'", - "version": "==3.14.0" + "version": "==3.16.0" }, "pycparser": { "hashes": [ @@ -632,30 +591,30 @@ }, "pypowsybl": { "hashes": [ - "sha256:06a9ac540861f1bf271838c6151a61d35886b3f89ab23a29dc88d1f42392d6b6", - "sha256:1015e4d0acc51634fe476a31a05e4b5827afe6cef2a5aa1a9b9ff97bba90c1a1", - "sha256:1f7106f3bdaf751446383ddd784c695210161889f77e1fe0e1994ae9616a3e24", - "sha256:4f06d9007395b92802a1f7ce891de915265bcff8aa6ac8baca4a93436e091ca3", - "sha256:4fd7e29245b913c6bd643ceb46d66a516a074147498412843922b3a56d1072f4", - "sha256:5db2abac09aec5feb0b6fc8d769fc788187ea3ac657e39c36d82238c59186621", - "sha256:7d6db29334de8bb9218db6919ff7ac0e9b3c159d957d229ac2a975f3229c7454", - "sha256:804b8a46989f303f5ef8851c2ba798fc7cf7b157b4ff3a9eb6a4e74bd117a5fa", - "sha256:8728f39fdc94cc22bae4e4189f0875cf0e2d7b8711c012522322a784a6d6a2b1", - "sha256:8974e5879450fa8505dccf5defe32d5b43b77b83e1a1237dbe7dd962db568095", - "sha256:8dc084734ddb34be73dd5ce727160a914eeb50d856ea112d3827a11ff3d6fb18", - "sha256:900ca4d292ee471d067e3552930dd29ba8e972ca49717644c142e03937fae0d3", - "sha256:9254f48c09e19fdef00f776f09c636976ee7373be7a2ef7209709993dadca40f", - "sha256:949752533c16126eaadec6ff2f9d9a1f101d5a31e145c4cb11fac5a106e3cb94", - "sha256:ce052c2b2bcf8fd044cab613bd220806d6d74cce3b4cf56842f6d0aa35115579", - "sha256:d76af6ee9ad031889e3356758c2d9e313c6cd5b625d6e72389a6c976e644f75a", - "sha256:d79df2aa4d6b4146aed7bc35be6092090740fa7c6a3e2dab09366dcc6e2619a2", - "sha256:d9ba5291da53b9c6660e3b80b9980486b23dc9976768c0e8f66338cc3a3a0ed3", - "sha256:f2bfb79da425483122b352f3a0ae636c1831dd25bcfdcad8354cda083c7e391a", - "sha256:f37cb5a3e4f74c06a642af9e33f0e07a7f7a6546bb5c223f51cddd73fc4eb73f" + "sha256:06eee912c5fd05ca0d21393a350f2355c22953767dd43d8f3defa51adfbdb858", + "sha256:23e628b62521dd46aa8e4b93aee5a58c366ace971d69bbd09775d1294c4d1f85", + "sha256:255c9e72a09c89fd0d99f98f02d34ca10f1d64108bd57059d4c7917c541db8b2", + "sha256:52cc308a921ae9221d093bcb01ebe7fc219753e91907d1d8b1c5910bc3da765b", + "sha256:54cda0f1b8115c01587a2fa73ecb76ce7b74435b60f017bed412da02bae7d75f", + "sha256:6325324a13fe2689063d2c96679d46517a05bc609358b6e502243f5572e14daf", + "sha256:6b575ff9b2ccb9d6dcb44f57d6824d00cf37293e31a7051eac27a03e7bf85708", + "sha256:776561013a13a113954b8aaa930b96935a8267897f2c570a1bd5045ce11c45fc", + "sha256:7da64de9f09906c0fba3d8caa9a5c32de9637c5b1c52016af885a49c4a7a1a4a", + "sha256:95dc568bb50db2be8e5fa5b3c5bdbf232f71db98ddd1869fd3a8429c9c73f948", + "sha256:a03ef2b6244388bdd869a2486646119ec2aa3efa49fc236603ab1ab161eb5ef9", + "sha256:a192d50800e4b08c5a04d72835243ffb64318b7652713fd4875ea318e51b6174", + "sha256:b1074d3ca73715521541200f2b8f75c0fd6450e26789013968022f4be2b12531", + "sha256:b5b94ed9c45d281130c7ab93258434e6e7856d1cb01c29b38e4ea67d4cdf5237", + "sha256:b7f52b7d01fb9b99245dc50a89322549eb953e74c3ad1ae95842cab648adc87a", + "sha256:c462c20b9a7ea1aef9930ec2c909fd54df7f269d7aa931628c1d68dc8b9209f8", + "sha256:c8cc64683e7c05a147dd1216aa912f301befcf2c752085b09ea5e074a06592ae", + "sha256:def9daf2c3410d49dda9ba97e4afea2254a6bf85c348c24db66ac471527ff99e", + "sha256:eaf992676631769b0dd1b009cfa041f001eacf1142ea4c087a6928942ba1fdb0", + "sha256:f8afb4255eedc060e81637fdc8943a3be60e52da241367030e2923632a586a08" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==1.9.0" + "version": "==1.11.2" }, "python-dateutil": { "hashes": [ @@ -667,20 +626,20 @@ }, "pytz": { "hashes": [ - "sha256:89dd22dca55b46eac6eda23b2d72721bf1bdfef212645d81513ef5d03038de57", - "sha256:c2db42be2a2518b28e65f9207c4d05e6ff547d1efa4086469ef855e4ab70178e" + "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", + "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00" ], "index": "pypi", - "version": "==2025.1" + "version": "==2025.2" }, "requests": { "hashes": [ - "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", - "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" + "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c", + "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==2.32.3" + "version": "==2.32.4" }, "requests-file": { "hashes": [ @@ -739,11 +698,11 @@ }, "triplets": { "hashes": [ - "sha256:3eb48071d266b890efe94a57ef93addda6acfd6cc5b6cd6cf9923bbe59dddbe3", - "sha256:73c116c51d23248ee7af5772625f222ea1489408fa1930f60ae748db9e913e68" + "sha256:6f728262a185508a316c243e2efa2fb0478e961a564cd261938811d0ff09d8bb", + "sha256:eddfa949ae5a32e280595ecfaea576b63deb8427cda830f28080f76de16f6c70" ], "index": "pypi", - "version": "==0.0.7" + "version": "==0.0.11" }, "typing-extensions": { "hashes": [ @@ -755,19 +714,19 @@ }, "tzdata": { "hashes": [ - "sha256:24894909e88cdb28bd1636c6887801df64cb485bd593f2fd83ef29075a81d694", - "sha256:7e127113816800496f027041c570f50bcd464a020098a3b6b199517772303639" + "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", + "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9" ], "markers": "python_version >= '2'", - "version": "==2025.1" + "version": "==2025.2" }, "urllib3": { "hashes": [ - "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472", - "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168" + "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760", + "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc" ], - "markers": "python_version >= '3.8'", - "version": "==2.2.2" + "markers": "python_version >= '3.9'", + "version": "==2.5.0" }, "wcwidth": { "hashes": [ @@ -778,19 +737,19 @@ }, "xmltodict": { "hashes": [ - "sha256:341595a488e3e01a85a9d8911d8912fd922ede5fecc4dce437eb4b6c8d037e56", - "sha256:aa89e8fd76320154a40d19a0df04a4695fb9dc5ba977cbb68ab3e4eb225e7852" + "sha256:201e7c28bb210e374999d1dde6382923ab0ed1a8a5faeece48ab525b7810a553", + "sha256:20cc7d723ed729276e808f26fb6b3599f786cbc37e06c65e192ba77c40f20aac" ], - "markers": "python_version >= '3.4'", - "version": "==0.13.0" + "markers": "python_version >= '3.6'", + "version": "==0.14.2" }, "zeep": { "hashes": [ - "sha256:6754feb4c34a4b6d65fbc359252bf6654dcce3937bf1d95aae4402a60a8f5939", - "sha256:72093acfdb1d8360ed400869b73fbf1882b95c4287f798084c42ee0c1ff0e425" + "sha256:a637aa7eedb6330bb27e8c94c5233ddf23553904323adf9398f8cf5025acb216", + "sha256:f45385e9e1b09d5550e0f51ab9fa7c6842713cab7194139372fd82a99c56a06e" ], - "markers": "python_version >= '3.7'", - "version": "==4.2.1" + "markers": "python_version >= '3.8'", + "version": "==4.3.1" } }, "develop": { diff --git a/config/cgm_worker/merger.properties b/config/cgm_worker/merger.properties index 1e636b23..b1a05de4 100644 --- a/config/cgm_worker/merger.properties +++ b/config/cgm_worker/merger.properties @@ -1,11 +1,13 @@ [MAIN] INPUT_RABBIT_QUE = emf-merge.tasks.rmm +OUTPUT_RMQ_EXCHANGE = emf-quality.test INPUT_MINIO_BUCKET = opde-confidential-models INPUT_MINIO_FOLDER = IGM OUTPUT_MINIO_BUCKET = opde-confidential-models OUTPUT_MINIO_FOLDER = EMFOS/RMM MERGE_REPORT_ELK_INDEX = emfos-merge-reports -SMALL_ISLAND_SIZE = 10 -MERGE_LOAD_FLOW_SETTINGS = CGM_RELAXED_3_CUSTOM -NET_INTERCHANGE_THRESHOLD = 200 -OPEN_NON_RETAINED_SWITCHES_BETWEEN_TN = True \ No newline at end of file +OPDE_MODELS_ELK_INDEX = emfos-opde-models +ENABLE_DYNAMIC_MERGE_SETTINGS = True +MERGE_LOAD_FLOW_SETTINGS = CGM_DEFAULT +MERGE_LOAD_FLOW_SETTINGS_PRIORITY = CGM_DEFAULT,CGM_RELAXED_1,CGM_RELAXED_2,CGM_RELAXED_3_CUSTOM +REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION = True \ No newline at end of file diff --git a/config/cgm_worker/post_processing.properties b/config/cgm_worker/post_processing.properties new file mode 100644 index 00000000..2b029c65 --- /dev/null +++ b/config/cgm_worker/post_processing.properties @@ -0,0 +1,5 @@ +[MAIN] +NET_INTERCHANGE_THRESHOLD = 200 +FIX_INJECTION_ERRORS = True +INJECTION_THRESHOLD = 0.1 +SMALL_ISLAND_SIZE = 10 \ No newline at end of file diff --git a/config/cgm_worker/replacement.properties b/config/cgm_worker/replacement.properties index 7713cf6f..5b2cf7e4 100644 --- a/config/cgm_worker/replacement.properties +++ b/config/cgm_worker/replacement.properties @@ -1,3 +1,2 @@ [MAIN] -REPLACEMENT_CONFIG = config/cgm_worker/replacement_config.json -QUERY_FILTER = now-2w \ No newline at end of file +REPLACEMENT_CONFIG = config/cgm_worker/replacement_config.json \ No newline at end of file diff --git a/config/cgm_worker/replacement_conf.json b/config/cgm_worker/replacement_conf.json index 3772c501..cf83d639 100644 --- a/config/cgm_worker/replacement_conf.json +++ b/config/cgm_worker/replacement_conf.json @@ -2,10 +2,13 @@ "configuration": { "validity_start": "2024-02-22T00:00:00Z", "validity_end": "2024-02-22T23:59:59Z", - "timeZone": "CET" + "time_zone": "CET" }, - "timeHorizon": { - "Request_list": ["ID", "1D", "2D", "WK", "1D", "MO", "YR"] + "time_horizon": { + "request_list": ["ID", "1D", "2D", "WK", "1D", "MO", "YR"] + }, + "replacement_length": { + "request_list": ["2w", "2w", "2w", "2w", "2w", "1m", "4m"] }, "month_ahead": { "hours": ["07:30", "08:30", "09:30", "06:30", "05:30", "04:30"], diff --git a/config/cgm_worker/scaler.properties b/config/cgm_worker/scaler.properties index 7225c735..12be5e26 100644 --- a/config/cgm_worker/scaler.properties +++ b/config/cgm_worker/scaler.properties @@ -1,4 +1,4 @@ [MAIN] -MAX_ITERATION = 10 +MAX_ITERATION = 15 BALANCE_THRESHOLD = 2 -DEBUG = True \ No newline at end of file +DEBUG = True diff --git a/config/integrations/object_storage.properties b/config/integrations/object_storage.properties index 5fc8d47e..7086de81 100644 --- a/config/integrations/object_storage.properties +++ b/config/integrations/object_storage.properties @@ -1,4 +1,3 @@ [MAIN] -ELASTIC_TASKS_INDEX = emfos-tasks ELASTIC_MODELS_INDEX = emfos-opde-models ELASTIC_SCHEDULES_INDEX = emfos-schedules \ No newline at end of file diff --git a/config/model_quality/model_quality.properties b/config/model_quality/model_quality.properties new file mode 100644 index 00000000..ddffd777 --- /dev/null +++ b/config/model_quality/model_quality.properties @@ -0,0 +1,4 @@ +[MAIN] +INPUT_RMQ_QUEUE = object-storage.models.quality +ELK_QUALITY_INDEX = emfos-model-quality +ELK_STATISTICS_INDEX = emfos-model-statistics \ No newline at end of file diff --git a/config/model_validator/model_validator.properties b/config/model_validator/model_validator.properties index d2d0ed35..8193b364 100644 --- a/config/model_validator/model_validator.properties +++ b/config/model_validator/model_validator.properties @@ -1,9 +1,13 @@ [MAIN] INPUT_RMQ_QUEUE = object-storage.models.validation +OUTPUT_RMQ_EXCHANGE = emf-quality.test VALIDATION_ELK_INDEX = emfos-igm-validation METADATA_ELK_INDEX = emfos-opde-models ELK_ID_FROM_METADATA_FIELDS = opde:Id,data-source ELK_ID_HASHING = True -VALIDATION_LOAD_FLOW_SETTINGS = CGM_RELAXED_2 +ENABLE_DYNAMIC_VALIDATION_SETTINGS = False +VALIDATION_LOAD_FLOW_SETTINGS = IGM_VALIDATION +VALIDATION_LOAD_FLOW_SETTINGS_PRIORITY = IGM_VALIDATION,CGM_DEFAULT,CGM_RELAXED_1,CGM_RELAXED_2,CGM_RELAXED_3_CUSTOM CHECK_NON_RETAINED_SWITCHES = False -CHECK_KIRCHHOFF_FIRST_LAW = False \ No newline at end of file +CHECK_KIRCHHOFF_FIRST_LAW = False +OPEN_NON_RETAINED_SWITCHES = True \ No newline at end of file diff --git a/config/task_generator/manual_task_generator.properties b/config/task_generator/manual_task_generator.properties index 1b27cbfa..aa26369f 100644 --- a/config/task_generator/manual_task_generator.properties +++ b/config/task_generator/manual_task_generator.properties @@ -14,12 +14,10 @@ TASK_REFERENCE_TIME = TASK_VERSION = TASK_MERGING_ENTITY = RUN_REPLACEMENT = -RUN_REPLACEMENT_LOCAL = RUN_SCALING = UPLOAD_TO_OPDM = UPLOAD_TO_MINIO = SEND_MERGE_REPORT = -PRE_TEMP_FIXES = POST_TEMP_FIXES = FORCE_OUTAGE_FIX = FIX_NET_INTERCHANGE2 = \ No newline at end of file diff --git a/config/task_generator/process_conf.json b/config/task_generator/process_conf.json index 386f8132..1c267498 100644 --- a/config/task_generator/process_conf.json +++ b/config/task_generator/process_conf.json @@ -312,6 +312,46 @@ "send_merge_report": "True" } }, + { + "@context": "https://example.com/schemas/run.jsonld", + "@id": "https://example.com/runs/IntraDayRMM/EOD", + "@type": "Run", + "process_id": "https://example.com/processes/RMM_CREATION", + "valid_from": "", + "valid_to": "", + "gate_open": "PT1H", + "gate_close": "PT45M", + "run_at": "00 15 29 02 3", + "time_frame": "ID", + "data_timestamps": "30 * * * *", + "data_resolution": "PT1H", + "tags": [], + "properties": { + "merge_type": "BA", + "merging_entity": "BALTICRCC", + "included": [ + "AST", + "PSE", + "ELERING", + "LITGRID" + ], + "excluded": [], + "local_import": [], + "time_horizon": "ID", + "version": "001", + "mas": "http://www.baltic-rsc.eu/OperationalPlanning/RMM", + "replacement": "True", + "replacement_local":"False", + "pre_temp_fixes":"True", + "post_temp_fixes":"True", + "fix_net_interchange2": "False", + "force_outage_fix":"False", + "scaling": "False", + "upload_to_opdm": "False", + "upload_to_minio": "True", + "send_merge_report": "True" + } + }, { "@context": "https://example.com/schemas/run.jsonld", "@id": "https://example.com/runs/DayAheadRMM", diff --git a/config/task_generator/task_generator.properties b/config/task_generator/task_generator.properties index b3a78214..b2e4754a 100644 --- a/config/task_generator/task_generator.properties +++ b/config/task_generator/task_generator.properties @@ -1,6 +1,6 @@ [MAIN] RMQ_EXCHANGE = emf-tasks -TASK_ELK_INDEX = +TASK_ELK_INDEX = emfos-tasks TASK_WINDOW_DURATION = PT1M TASK_WINDOW_REFERENCE = currentMinuteStart TIMESTAMP = diff --git a/docker/Dockerfile b/docker/Dockerfile index 233a83c6..a53e68e4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,17 +1,17 @@ # Define base container image (platform has to be defined if using with ARM64 architecture) -FROM python:3.11 as emfos-base -#FROM --platform=linux/amd64 python:3.11 as emfos-base +FROM python:3.11-slim as emfos-base +#FROM --platform=linux/amd64 python:3.11-slim as emfos-base # Set working directory WORKDIR /src # Install tools to set up python environment +RUN apt-get update && apt-get install -y --no-install-recommends procps && rm -rf /var/lib/apt/lists/* RUN pip install --upgrade setuptools pip uv pipenv # Install needed modules COPY Pipfile . COPY Pipfile.lock . -#COPY requirements.txt . #RUN PIPENV_VENV_IN_PROJECT=1 pipenv install --deploy #RUN pipenv install --system --deploy RUN pipenv requirements > requirements.txt @@ -73,7 +73,6 @@ WORKDIR /src # Copy files to container COPY emf/model_merger/ ./emf/model_merger/ COPY emf/common/ ./emf/common/ -COPY emf/task_generator/ ./emf/task_generator/ COPY config/ ./config/ # Start process @@ -95,3 +94,15 @@ COPY config/ ./config/ ENV PYTHONPATH="${PYTHONPATH}:/src" ENTRYPOINT ["python", "emf/task_generator/worker.py"] #ENTRYPOINT ["/bin/bash"] + +# Model Quality image +FROM balticrcc/emfos-base:latest as emfos-model-quality + +WORKDIR /src + +COPY emf/model_quality/ ./emf/model_quality/ +COPY emf/common/ ./emf/common/ +COPY config/ ./config/ + +ENV PYTHONPATH="${PYTHONPATH}:/src" +ENTRYPOINT ["python", "emf/model_quality/worker.py"] diff --git a/emf/common/decorators.py b/emf/common/decorators.py index 16c3be60..37eaa4bb 100644 --- a/emf/common/decorators.py +++ b/emf/common/decorators.py @@ -5,6 +5,7 @@ # Start logger logger = logging.getLogger(__name__) + def performance_counter(units='seconds'): """Counts performance of the function""" def decorator_performance_counter(func): @@ -15,7 +16,7 @@ def wrapper_performance_counter(*args, **kwargs): duration = round(time.perf_counter() - start_time, 2) if units == 'minutes': duration = round(duration / 60, 2) # counting by minutes - logger.info(f"Process {func.__name__!r} finished with duration: {duration} {units}", extra={'numeric_value': duration}) + logger.info(f"Process {func.__name__!r} finished with duration: {duration} {units}") return response return wrapper_performance_counter return decorator_performance_counter diff --git a/emf/common/helpers/cgmes.py b/emf/common/helpers/cgmes.py new file mode 100644 index 00000000..527abaed --- /dev/null +++ b/emf/common/helpers/cgmes.py @@ -0,0 +1,53 @@ +import logging +import json +import pandas as pd +import config +from lxml import etree + +logger = logging.getLogger(__name__) + + +def export_to_cgmes_zip(triplets: list): + namespace_map = { + "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + "cim": "http://iec.ch/TC57/2013/CIM-schema-cim16#", + "md": "http://iec.ch/TC57/61970-552/ModelDescription/1#", + "entsoe": "http://entsoe.eu/CIM/SchemaExtension/3/1#", + } + + rdf_map = json.load(config.paths.cgm_worker.CGMES_v2_4_15_2014_08_07) + + return pd.concat(triplets, ignore_index=True).export_to_cimxml(rdf_map=rdf_map, + namespace_map=namespace_map, + export_undefined=False, + export_type="xml_per_instance_zip_per_xml", + debug=False, + export_to_memory=True) + + +def get_metadata_from_rdfxml(parsed_xml: etree._ElementTree): + """Parse model metadata form xml, returns a dictionary with metadata""" + + assert isinstance(parsed_xml, etree._ElementTree) + + rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" + + header = parsed_xml.find("{*}FullModel") # Version update proof, as long as element name remains the same + meta_elements = header.getchildren() + + # Add model ID from FullModel@about + metadata = {"Model.mRID": header.attrib.get(f'{{{rdf}}}about').split(":")[-1]} + + # Add all other metadata + for element in meta_elements: + key = element.tag.split("}")[1] + value = element.text if element.text else element.attrib.get(f"{{{rdf}}}resource") + if existing_value := metadata.get(key): + value = existing_value + [value] if isinstance(existing_value, list) else [existing_value, value] + metadata[key] = value + + return metadata + + +if __name__ == "__main__": + pass \ No newline at end of file diff --git a/emf/common/helpers/loadflow.py b/emf/common/helpers/loadflow.py new file mode 100644 index 00000000..314505ca --- /dev/null +++ b/emf/common/helpers/loadflow.py @@ -0,0 +1,207 @@ +import logging +import uuid +import re +import pandas as pd +from io import BytesIO +from zipfile import ZipFile, ZIP_DEFLATED +import pypowsybl +from typing import List + +logger = logging.getLogger(__name__) + + +def package_for_pypowsybl(opdm_objects, return_zip: bool = False): + """ + Method to transform OPDM objects into sufficient format binary buffer or zip package + :param opdm_objects: list of OPDM objects + :param return_zip: flag to save OPDM objects as zip package in local directory + :return: binary buffer or zip package file name + """ + output_object = BytesIO() + if return_zip: + output_object = f"{uuid.uuid4()}.zip" + logger.info(f"Adding files to {output_object}") + + with ZipFile(output_object, "w") as global_zip: + for opdm_components in opdm_objects: + for instance in opdm_components['opde:Component']: + with ZipFile(BytesIO(instance['opdm:Profile']['DATA'])) as instance_zip: + for file_name in instance_zip.namelist(): + logger.info(f"Adding file: {file_name}") + global_zip.writestr(file_name, instance_zip.open(file_name).read()) + + return output_object + + +def load_network_model(opdm_objects: List[dict], parameters: dict = None, skip_default_parameters: bool = False): + """ + Loads given list of models (opdm_objects) into pypowsybl using internal (known good) default_parameters + Additional parameters can be specified as a dict in field parameters which will overwrite the default ones if keys + are matching + :param opdm_objects: list of dictionaries following the opdm model format + :param parameters: dictionary of desired parameters for loading models to pypowsybl + :param skip_default_parameters: skip the default parameters + """ + default_parameters = {"iidm.import.cgmes.import-node-breaker-as-bus-breaker": 'true'} + if not skip_default_parameters: + if not parameters: + parameters = default_parameters + else: + # Give a priority to parameters given from outside + parameters = {**default_parameters, **parameters} + + import_report = pypowsybl.report.Reporter() + network = pypowsybl.network.load_from_binary_buffer( + buffer=package_for_pypowsybl(opdm_objects), + reporter=import_report, + parameters=parameters + # parameters={ + # "iidm.import.cgmes.store-cgmes-model-as-network-extension": 'true', + # "iidm.import.cgmes.create-active-power-control-extension": 'true', + # "iidm.import.cgmes.post-processors": ["EntsoeCategory"]} + ) + + logger.info(f"Loaded: {network}") + logger.debug(f"{import_report}") + + return network + + +def get_network_elements(network: pypowsybl.network, + element_type: pypowsybl.network.ElementType, + all_attributes: bool = True, + attributes: List[str] = None, + **kwargs + ): + + _voltage_levels = network.get_voltage_levels(all_attributes=True).rename(columns={"name": "voltage_level_name"}) + _substations = network.get_substations(all_attributes=True).rename(columns={"name": "substation_name"}) + + elements = network.get_elements(element_type=element_type, all_attributes=all_attributes, attributes=attributes, **kwargs) + elements = elements.merge(_voltage_levels, left_on='voltage_level_id', right_index=True, suffixes=(None, '_voltage_level')) + elements = elements.merge(_substations, left_on='substation_id', right_index=True, suffixes=(None, '_substation')) + + # Need to ensure that column 'isHvdc' is present if DANGLING_LINE type is requested + if element_type is pypowsybl.network.ElementType.DANGLING_LINE: + if 'isHvdc' not in elements.columns: + elements['isHvdc'] = '' + + return elements + + +def get_slack_generators(network: pypowsybl.network): + slack_terminals = network.get_extension('slackTerminal') + slack_generators = get_network_elements(network=network, + element_type=pypowsybl.network.ElementType.GENERATOR, + all_attributes=True, + id=slack_terminals['element_id']) + + return slack_generators + + +def get_connected_components_data(network: pypowsybl.network, + bus_count_threshold: int | None = None, + country_col_name: str = 'country'): + buses = get_network_elements(network, pypowsybl.network.ElementType.BUS) + data = buses.groupby('connected_component').agg(countries=(country_col_name, lambda x: list(x.unique())), + bus_count=('name', 'size')) + if bus_count_threshold: + data = data[data.bus_count > bus_count_threshold] + + return data.to_dict('index') + + +def get_model_outages(network: pypowsybl.network): + + # Get network elements + lines = network.get_elements(element_type=pypowsybl.network.ElementType.LINE, all_attributes=True).reset_index(names=['grid_id']) + _voltage_levels = network.get_voltage_levels(all_attributes=True).rename(columns={"name": "voltage_level_name"}) + _substations = network.get_substations(all_attributes=True).rename(columns={"name": "substation_name"}) + lines = lines.merge(_voltage_levels, left_on='voltage_level1_id', right_index=True, suffixes=(None, '_voltage_level')) + lines = lines.merge(_substations, left_on='substation_id', right_index=True, suffixes=(None, '_substation')) + lines['element_type'] = 'LINE' + + dlines = get_network_elements(network, pypowsybl.network.ElementType.DANGLING_LINE).reset_index(names=['grid_id']) + dlines['element_type'] = 'DANGLING_LINE' + + gens = get_network_elements(network, pypowsybl.network.ElementType.GENERATOR).reset_index(names=['grid_id']) + gens['element_type'] = 'GENERATOR' + + # Filter out disconnected elements + disconnected_lines = lines[(lines['connected1'] == False) | (lines['connected2'] == False)] + disconnected_dlines = dlines[dlines['connected'] == False] + disconnected_gens = gens[gens['connected'] == False] + + # Filter out only 330kv and above + disconnected_elements = pd.concat([disconnected_lines, disconnected_dlines, disconnected_gens]) + disconnected_elements = disconnected_elements[disconnected_elements['nominal_v'] >= 330] + + return disconnected_elements.to_dict('records') + + +def parse_pypowsybl_report(report: str): + lines = report.replace('+', '').splitlines() + all_network_dicts = [] + + current_dict = None + base_indent = None + + for line in lines: + stripped_line = line.strip() + + # Identify "Network info" line and its indentation level + if "Network info" in stripped_line: + if current_dict is not None: + # Save the current dictionary if a new "Network info" block starts + all_network_dicts.append(current_dict) + + current_dict = {} + base_indent = len(line) - len(stripped_line) + continue + + if current_dict is not None: + # Calculate the current line's indentation level relative to "Network info" + current_indent = len(line) - len(line.lstrip()) + + # Check for the specific phrase "Network has x buses and y branches" + match = re.match(r"Network has (\d+) buses and (\d+) branches", stripped_line) + if match: + buses = int(match.group(1)) + branches = int(match.group(2)) + current_dict['buses'] = buses + current_dict['branches'] = branches + + # Process lines with key-value pairs after ':' + elif ':' in stripped_line: + dict_name, key_values = stripped_line.split(':', 1) + dict_name = dict_name.strip() + key_values = key_values.strip() + + # Parse key-value pairs + if '=' in key_values: + current_dict[dict_name] = {} + for pair in key_values.split(','): + key, value = map(str.strip, pair.split('=')) + current_dict[dict_name][key] = value + else: + # Handle plain strings after ':' + current_dict[dict_name] = key_values + + else: + # Stop processing this block if indentation level is not greater than base_indent + if current_indent <= base_indent and current_dict: + all_network_dicts.append(current_dict) + current_dict = None + + # Append the last dictionary if it exists + if current_dict is not None and current_dict: + all_network_dicts.append(current_dict) + + # Filter out empty dicts + result = [n for n in all_network_dicts if n] + + return result + + +if __name__ == "__main__": + pass \ No newline at end of file diff --git a/emf/common/helpers/opdm_objects.py b/emf/common/helpers/opdm_objects.py new file mode 100644 index 00000000..cf6aa868 --- /dev/null +++ b/emf/common/helpers/opdm_objects.py @@ -0,0 +1,220 @@ +import logging +from io import BytesIO +import pandas as pd +import triplets +from lxml import etree +from emf.common.helpers.time import parse_datetime +from emf.common.helpers.utils import get_xml_from_zip +from emf.common.helpers.cgmes import get_metadata_from_rdfxml + + +logger = logging.getLogger(__name__) + + +def clean_data_from_opdm_objects(opdm_objects: list) -> list: + for opdm_object in opdm_objects: + for component in opdm_object['opde:Component']: + component['opdm:Profile']['DATA'] = None + + return opdm_objects + + +def save_opdm_objects(opdm_objects: dict) -> list: + """ + Function save OPDM objects on to local filesystem + :param opdm_objects: list of OPDM objects + :return: list of exported files + """ + exported_files = [] + for opdm_components in opdm_objects: + for instance in opdm_components['opde:Component']: + file_name = instance['opdm:Profile']['pmd:fileName'] + logger.info(f'Saving: {file_name}') + with open(file_name, 'wb') as instance_zip: + instance_zip.write(instance['opdm:Profile']['DATA']) + exported_files.append(file_name) + + return exported_files + + +def get_opdm_component_data_bytes(opdm_component: dict): + data = BytesIO(opdm_component['opdm:Profile']['DATA']) + data.name = opdm_component['opdm:Profile']['pmd:fileName'] + return data + + +def load_opdm_objects_to_triplets(opdm_objects: list[dict], profile: str | None = None): + if profile: + return pd.read_RDF([get_opdm_component_data_bytes(instance) for model in opdm_objects for instance in model['opde:Component'] if instance['opdm:Profile']['pmd:cgmesProfile'] == profile]) + return pd.read_RDF([get_opdm_component_data_bytes(instance) for model in opdm_objects for instance in model['opde:Component']]) + + +def get_metadata_from_file_name(file_name: str, meta_separator: str = "_"): + """ + Parse OPDM metadata from a filename string into a dictionary. + + Args: + :param file_name: (str): The filename containing metadata separated by underscores + and a file extension + :param meta_separator: (str): How the elements are separated in the string, usually by "_" + + Returns: + dict: Dictionary containing parsed metadata with OPDM-specific keys. + + Raises: + AssertionError: If file_name is not a string. + ValueError: If metadata parsing fails due to incorrect format. + """ + # Constants for OPDM metadata keys + VALID_FROM = 'pmd:validFrom' + TIME_HORIZON = 'pmd:timeHorizon' + CGMES_PROFILE = 'pmd:cgmesProfile' + VERSION_NUMBER = 'pmd:versionNumber' + MERGING_ENTITY = 'pmd:mergingEntity' + MERGING_AREA = 'pmd:mergingArea' + MODEL_PART = 'pmd:modelPartReference' + TSO = "pmd:TSO" + SOURCING_ACTOR = "pmd:sourcingActor" + + # Constant for file type + FILE_TYPE = 'file_type' + + # Input validation + assert isinstance(file_name, str), "file_name must be a string" + + metadata = {} # Meta container + + # Split filename into name and extension + file_name, metadata[FILE_TYPE] = file_name.split(".") + meta_list = file_name.split(meta_separator) + + # Parse main metadata components + if len(meta_list) == 4: + metadata[VALID_FROM], model_authority, metadata[CGMES_PROFILE], metadata[VERSION_NUMBER] = meta_list + metadata[TIME_HORIZON] = "" + elif len(meta_list) == 5: + metadata[VALID_FROM], metadata[TIME_HORIZON], model_authority, metadata[CGMES_PROFILE], metadata[VERSION_NUMBER] = meta_list + else: + logger.warning(f"Parsing error, number of allowed meta in filename is 4 or 5 separated by {meta_separator}: {file_name}") + return metadata + + # Parse model authority components + model_authority_list = model_authority.split("-") + + if len(model_authority_list) == 1: + metadata[MODEL_PART] = model_authority + elif len(model_authority_list) == 2: + metadata[MERGING_ENTITY], metadata[MERGING_AREA] = model_authority_list + elif len(model_authority_list) == 3: + metadata[MERGING_ENTITY], metadata[MERGING_AREA], metadata[MODEL_PART] = model_authority_list + else: + logger.error(f"Parsing error {model_authority}") + + # Add aliases + if metadata.get(MODEL_PART): + metadata[TSO] = metadata[MODEL_PART] + metadata[SOURCING_ACTOR] = metadata[MODEL_PART] + + return metadata + + +def generate_opdm_object_content_reference_from_filename(file_name: str, opdm_object_type: str = "CGMES"): + """ + Generates the file path based on the given parameters and metadata extracted from the filename. + + Parameters: + file_name (str): The name of the file from which metadata will be extracted. + opdm_object_type (str): The type of OPDM object, defaults to "CGMES". + + Returns: + str: Generated file path using the provided template and extracted metadata. + """ + template = "{opdm_object_type}/{timeHorizon}/{sourcingActor}/{validFrom:%Y%m%d}/{validFrom:%H%M00}/{cgmesProfile}/{file_name}" + + meta = {key.split(":")[-1]: value for key, value in get_metadata_from_file_name(file_name).items()} + meta["validFrom"] = parse_datetime(meta["validFrom"]) + meta["file_name"] = file_name + meta["opdm_object_type"] = opdm_object_type + + return template.format(**meta) + + +def get_opdm_metadata_from_rdfxml(parsed_xml: etree._ElementTree): + opdm_metadata_map = { + "Model.mRID": ["opde:Id", "pmd:modelid", "pmd:fullModel_ID"], + "Model.scenarioTime": ["pmd:scenarioDate"], + "Model.created": ["pmd:creationDate"], + "Model.description": ["pmd:description"], + "Model.version": ["pmd:version"], + "Model.DependentOn": ["opde:DependsOn"], + "Model.profile": ["pmd:modelProfile"], + "Model.modelingAuthoritySet": ["pmd:modelingAuthoritySet"], + + } + + metadata = {} + for key, value in get_metadata_from_rdfxml(parsed_xml).items(): + if opdm_keys := opdm_metadata_map.get(key): + for opdm_key in opdm_keys: + metadata[opdm_key] = value + + return metadata + + +def create_opdm_objects(models: list, metadata: dict | None = None, key_profile: str = "SV") -> list: + """ + Function to create OPDM object like structure in memory + input models, is a nested list [[EQ, SSH, TP, SV], [EQ, SSH, TP, SC]] where upper list will become opdm:Object + and nested will be opdm:Profile + + :return: list of OPDM objects + """ + opdm_objects = [] + + for model in models: + opdm_object = {'opde:Component': []} + + for profile_instance in model: + + # Build profile instance metadata + opdm_profile = get_metadata_from_file_name(profile_instance.name) + opdm_profile.update(get_opdm_metadata_from_rdfxml(get_xml_from_zip(profile_instance))) + opdm_profile['pmd:fileName'] = profile_instance.name + opdm_profile["pmd:content-reference"] = generate_opdm_object_content_reference_from_filename(profile_instance.name) + + # Check if key profile and add to main object metadata + if opdm_profile.get('pmd:cgmesProfile') == key_profile: + opdm_object.update(opdm_profile) + + # Add model type - IGM/CGM + # TODO develop logic to also define as CGM metadata object + opdm_object['opde:Object-Type'] = "IGM" + + # Add DATA + opdm_profile['DATA'] = profile_instance.getvalue() + + # Add component to main object + opdm_object['opde:Component'].append({'opdm:Profile': opdm_profile}) + + # If metadata provided, overwrite existing + if metadata: + opdm_object.update(metadata) + + opdm_objects.append(opdm_object) + + return opdm_objects + + +def filename_from_opdm_metadata(metadata: dict, file_type: str | None = None): + model_authority = f"{metadata['pmd:mergingEntity']}-{metadata['pmd:Area']}" + file_name = f"{metadata['pmd:validFrom']}_{metadata['pmd:timeHorizon']}_{model_authority}_{metadata['pmd:cgmesProfile']}_{metadata['pmd:versionNumber']}" + + if file_type: + file_name = f"{file_name}.{file_type}" + + return file_name + + +if __name__ == "__main__": + # Create OPDM objects + opdm_object = create_opdm_objects([rdfzip_files]) \ No newline at end of file diff --git a/emf/common/helpers/tasks.py b/emf/common/helpers/tasks.py new file mode 100644 index 00000000..0044de32 --- /dev/null +++ b/emf/common/helpers/tasks.py @@ -0,0 +1,38 @@ +import logging +from datetime import datetime +import config +from emf.common.integrations.elastic import Elastic +from emf.common.config_parser import parse_app_properties + +logger = logging.getLogger(__name__) +parse_app_properties(globals(), config.paths.task_generator.task_generator) + + +def update_task_status(task: dict, status_text: str, publish: bool = True): + """Update task status + Will update task_update_time + Will update task_status + Will append new status to task_status_trace""" + + logger.info(f"Updating task status to: {status_text}") + + utc_now = datetime.utcnow().isoformat() + + task["task_update_time"] = utc_now + task["task_status"] = status_text + + task["task_status_trace"].append({ + "status": status_text, + "timestamp": utc_now + }) + + # TODO - better handling if elk is not available, possibly set elk connection timeout really small or refactor the sending to happen via rabbit + if publish: + try: + Elastic.send_to_elastic(index=TASK_ELK_INDEX, json_message=task, id=task.get("@id")) + except Exception as e: + logger.error(f"Task publication to Elastic failed with error: {e}") + + +if __name__ == "__main__": + pass \ No newline at end of file diff --git a/emf/common/time_helper.py b/emf/common/helpers/time.py similarity index 99% rename from emf/common/time_helper.py rename to emf/common/helpers/time.py index fda6f096..81d30397 100644 --- a/emf/common/time_helper.py +++ b/emf/common/helpers/time.py @@ -235,6 +235,7 @@ def convert_to_utc(date_time, default_timezone='Europe/Brussels'): return date_time.astimezone(timezone("UTC")) + def convert_to_timezone(date_time, from_timezone="UTC", to_timezone='Europe/Brussels'): """ Converts a datetime object from timezone to timezone. @@ -259,6 +260,7 @@ def convert_to_timezone(date_time, from_timezone="UTC", to_timezone='Europe/Brus def utcnow(): return datetime.now(timezone("UTC")) + def parse_datetime(iso_string, keep_timezone=True): if keep_timezone: diff --git a/emf/common/helpers/utils.py b/emf/common/helpers/utils.py new file mode 100644 index 00000000..dca06a24 --- /dev/null +++ b/emf/common/helpers/utils.py @@ -0,0 +1,121 @@ +import logging +from datetime import datetime +from io import BytesIO +from zipfile import ZipFile, ZIP_DEFLATED +from lxml import etree + +logger = logging.getLogger(__name__) + + +def attr_to_dict(instance: object, sanitize_to_strings: bool = False): + """ + Method to return class variables/attributes as dictionary + pypowsybl._pypowsybl.LimitViolation -> dict + :param instance: class instance + :param sanitize_to_strings: flag to convert attributes to string type + :return: dict + """ + + def convert_value(value): + if isinstance(value, datetime): + return value.isoformat() if sanitize_to_strings else value + elif isinstance(value, list): + return [convert_value(item) for item in value] + elif isinstance(value, dict): + return {k: convert_value(v) for k, v in value.items()} + elif hasattr(value, '__dict__'): + return attr_to_dict(value, sanitize_to_strings) + elif sanitize_to_strings: + return str(value) + return value + + attribs = [attr for attr in dir(instance) if (not callable(getattr(instance, attr)) and not attr.startswith("_"))] + result_dict = {attr_key: convert_value(getattr(instance, attr_key)) for attr_key in attribs} + + return result_dict + + +def flatten_dict(nested_dict: dict, parent_key: str = '', separator: str = '.'): + """ + Flattens a nested dictionary. + + Parameters: + - nested_dict (dict): The dictionary to flatten. + - parent_key (str): The base key string used for recursion. + - separator (str): The separator between parent and child keys. + + Returns: + - dict: A flattened dictionary where nested keys are concatenated into a single string. + """ + items = [] + for k, v in nested_dict.items(): + new_key = f"{parent_key}{separator}{k}" if parent_key else k + if isinstance(v, dict): + items.extend(flatten_dict(v, new_key, separator=separator).items()) + elif isinstance(v, list): + for i, item in enumerate(v): + if isinstance(item, dict): + items.extend(flatten_dict(item, f"{new_key}[{i}]", separator=separator).items()) + else: + items.append((f"{new_key}[{i}]", item)) + else: + items.append((new_key, v)) + return dict(items) + + +def convert_dict_str_to_bool(data_dict: dict): + for key, value in data_dict.items(): + if isinstance(value, str): + if value in ['True', 'true', 'TRUE']: + data_dict[key] = True + elif value in ['False', 'false', 'FALSE']: + data_dict[key] = False + elif isinstance(value, dict): + # Recursively converter nested dictionaries + data_dict[key] = convert_dict_str_to_bool(value) + + return data_dict + + +def filter_and_flatten_dict(nested_dict: dict, keys: list): + """ + Creates a new flat dictionary from specified keys. + + Parameters: + - nested_dict (dict): The original nested dictionary. + - keys (list): The list of keys to include in the new flat dictionary. + + Returns: + - dict: A new flat dictionary with only the specified keys. + """ + flattened = flatten_dict(nested_dict) + return {key: flattened[key] for key in keys if key in flattened} + + +def zip_xml(xml_file_object: BytesIO): + xml_file_name = xml_file_object.name + xml_file_extension = xml_file_name.split(".")[-1] + zip_file_name = xml_file_name.replace(f".{xml_file_extension}", ".zip") + + zip_file_object = BytesIO() + zip_file_object.name = zip_file_name + + # Create and save ZIP + with ZipFile(zip_file_object, 'w', ZIP_DEFLATED) as zipfile: + zipfile.writestr(xml_file_name, xml_file_object.getvalue()) + + return zip_file_object + + +def get_xml_from_zip(zip_file_path: str): + with ZipFile(zip_file_path, 'r') as zipfile_object: + xml_file_name = zipfile_object.namelist()[0] + file_bytes = zipfile_object.read(xml_file_name) + # Convert bytes to file-like object for parsing + xml_tree_object = etree.parse(BytesIO(file_bytes)) + + return xml_tree_object + + +if __name__ == "__main__": + pass \ No newline at end of file diff --git a/emf/common/integrations/elastic.py b/emf/common/integrations/elastic.py index 1238d2b3..adf7a658 100644 --- a/emf/common/integrations/elastic.py +++ b/emf/common/integrations/elastic.py @@ -149,6 +149,10 @@ def update_document(self, index: str, id: str, body: dict): def get_docs_by_query(self, index: str, query: dict, size: int | None = None, return_df: bool = True): + # Validate index definition to be able to search all index by pattern + if "*" not in index: + index = f"{index}*" + response = self.client.search(index=index, query=query, size=size) if self.debug: logger.info(f"Returned total {response['hits']['total']['value']} document") diff --git a/emf/common/integrations/minio_api.py b/emf/common/integrations/minio_api.py index d48517a3..3e01732e 100644 --- a/emf/common/integrations/minio_api.py +++ b/emf/common/integrations/minio_api.py @@ -2,6 +2,7 @@ import requests from lxml import etree import minio +from minio.commonconfig import Tags import urllib3 import sys import mimetypes @@ -15,7 +16,7 @@ from datetime import datetime, timedelta from aniso8601 import parse_datetime from emf.common.config_parser import parse_app_properties -from emf.common.loadflow_tool.helper import opdm_metadata_from_filename +from emf.common.helpers.opdm_objects import get_metadata_from_file_name urllib3.disable_warnings() logger = logging.getLogger(__name__) @@ -90,13 +91,27 @@ def _get_credentials(self, action: str = "AssumeRoleWithLDAPIdentity", version: return credentials + @staticmethod + def dict_to_tags(tags: dict): + converted = Tags.new_object_tags() + for k, v in tags.items(): + converted[k] = v + + return converted + @renew_authentication_token - def upload_object(self, file_path_or_file_object: str | BytesIO, bucket_name: str, metadata: dict | None = None): + def upload_object(self, + file_path_or_file_object: str | BytesIO, + bucket_name: str, + metadata: dict | None = None, + tags: dict | None = None, + ): """ Method to upload file to Minio storage :param file_path_or_file_object: file path or BytesIO object :param bucket_name: bucket name :param metadata: object metadata + :param tags: object tags :return: response from Minio """ file_object = file_path_or_file_object @@ -107,6 +122,10 @@ def upload_object(self, file_path_or_file_object: str | BytesIO, bucket_name: st else: length = file_object.getbuffer().nbytes + # Handle tags if provided + if tags: + tags = self.dict_to_tags(tags) + # Just to be sure that pointer is at the beginning of the content file_object.seek(0) @@ -118,7 +137,8 @@ def upload_object(self, file_path_or_file_object: str | BytesIO, bucket_name: st data=file_object, length=length, content_type=mimetypes.guess_type(file_object.name)[0], - metadata=metadata + metadata=metadata, + tags=tags, ) return response @@ -188,16 +208,16 @@ def query_objects(self, bucket_name: str, metadata: dict = None, prefix: str = N return result_list def get_all_objects_name(self, bucket_name: str, prefix: str = None): - objects = self.client.list_objects(bucket_name=bucket_name, prefix=prefix,recursive=True) + objects = self.client.list_objects(bucket_name=bucket_name, prefix=prefix, recursive=True) list_elements = [] for obj in objects: try: object_name = obj.object_name.split("/")[-1] - #only take models that have metadata in the filename + # Only take models that have metadata in the filename if len(object_name.split('-')) > 3: list_elements.append(object_name) - except: - logger.warning(f"Object name not present") + except Exception as error: + logger.warning(f"Object name not present, exception: {error}") return list_elements @@ -283,7 +303,7 @@ def get_latest_models_and_download(self, "pmd:fileName": file_name, "DATA": source_zip.open(file_name).read()} - metadata.update(opdm_metadata_from_filename(file_name)) + metadata.update(get_metadata_from_file_name(file_name)) opdm_profile = {'opdm:Profile': metadata} opdm_object['opde:Component'].append(opdm_profile) diff --git a/emf/common/integrations/object_storage/models.py b/emf/common/integrations/object_storage/models.py index 657ae99a..2d63b4c1 100644 --- a/emf/common/integrations/object_storage/models.py +++ b/emf/common/integrations/object_storage/models.py @@ -8,6 +8,74 @@ logger = logging.getLogger(__name__) +def compile_query(metadata: dict, filter: str | None): + + match_and_term_list = [] + for key, value in metadata.items(): + if isinstance(value, list): + match_and_term_list.append({"terms": {key: value}}) + else: + match_and_term_list.append({"match": {key: value}}) + + if filter: + query = {"bool": {"must": match_and_term_list, "filter": {"range": {"pmd:scenarioDate": {"gte": filter}}}}} + else: + query = {"bool": {"must": match_and_term_list}} + + return query + + +def fetch_unique_values(metadata_query: dict, + field: str, + query_filter: str | None = None, + index: str = object_storage.ELASTIC_MODELS_INDEX, + page_size: int = 10000, + ): + + # Validate index definition to be able to search all index by pattern + if "*" not in index: + index = f"{index}*" + + agg_name = "values_page" + unique_vals = [] + after_key = None + + # Compile query + query = compile_query(metadata=metadata_query, filter=query_filter) + + while True: + comp = { + "size": page_size, + "sources": [ + {"val": {"terms": {"field": field}}} + ] + } + if after_key: + comp["after"] = after_key + + body = { + "size": 0, + "query": query, + "aggs": { + agg_name: { + "composite": comp + } + } + } + + response = object_storage.elastic_service.client.search(index=index, body=body) + buckets = response["aggregations"][agg_name]["buckets"] + unique_vals.extend(b["key"]["val"] for b in buckets) + + # Page on if there's more + if "after_key" in response["aggregations"][agg_name]: + after_key = response["aggregations"][agg_name]["after_key"] + else: + break + + return unique_vals + + def query_data(metadata_query: dict, query_filter: str | None = None, index: str = object_storage.ELASTIC_MODELS_INDEX, @@ -53,23 +121,14 @@ def query_data(metadata_query: dict, if "*" not in index: index = f"{index}*" - match_and_term_list = [] - for key, value in metadata_query.items(): - if isinstance(value, list): - match_and_term_list.append({"terms": {key: value}}) - else: - match_and_term_list.append({"match": {key: value}}) - - if query_filter: - query = {"bool": {"must": match_and_term_list, "filter": {"range": {"pmd:scenarioDate": {"gte": query_filter}}}}} - else: - query = {"bool": {"must": match_and_term_list}} + # Compile query + query = compile_query(metadata=metadata_query, filter=query_filter) # Return query results response = object_storage.elastic_service.client.search(index=index, query=query, size=size, sort=sort, scroll=scroll) scroll_id = response['_scroll_id'] hits = response["hits"]["hits"] - content_list = [content["_source"] for content in hits] + content_list = [content["_source"] for content in hits] # TODO - consider optimize without this loop while len(hits) > 0: response = object_storage.elastic_service.client.scroll(scroll_id=scroll_id, scroll=scroll) hits = response["hits"]["hits"] @@ -107,7 +166,7 @@ def get_content(metadata: dict): components_received = [] for component in metadata["opde:Component"]: content_reference = component.get("opdm:Profile").get("pmd:content-reference") - logger.info(f"Downloading object: {content_reference}") + logger.info(f"Downloading object: {bucket_name}/{content_reference}") content = object_storage.minio_service.download_object(bucket_name, content_reference) component["opdm:Profile"]["DATA"] = content components_received.append(bool(content)) # collect boolean flags of received components diff --git a/emf/common/integrations/object_storage/tasks.py b/emf/common/integrations/object_storage/tasks.py deleted file mode 100644 index bfb64993..00000000 --- a/emf/common/integrations/object_storage/tasks.py +++ /dev/null @@ -1,20 +0,0 @@ -from emf.common.integrations.object_storage import * -from emf.common.integrations.object_storage.models import query_data -from functools import partial -import logging -logger = logging.getLogger(__name__) - - -def publish_tasks(tasks: list, index=ELASTIC_TASKS_INDEX): - - for task in tasks: - logger.info(f"Adding task to {ELASTIC_TASKS_INDEX}") - elastic_service.send_to_elastic( - index=index, - json_message=task, - id=task.get("@id") - ) - - -query_tasks = partial(query_data, index=ELASTIC_TASKS_INDEX) - diff --git a/emf/common/integrations/opdm.py b/emf/common/integrations/opdm.py index 219b5903..511ec640 100644 --- a/emf/common/integrations/opdm.py +++ b/emf/common/integrations/opdm.py @@ -18,76 +18,77 @@ class OPDM(opdm_api.create_client): def __init__(self, server=OPDM_SERVER, username=OPDM_USERNAME, password=OPDM_PASSWORD, debug=False, verify=False): super().__init__(server, username, password, debug, verify) - # TODO - update query function in OPDM module - def query(self, object_type, meta=None): + def query(self, object_type, meta = None): - logger.info(f"Sending query to OPDM for {object_type} with parameters {meta}") + logger.info(f"Sending query to OPDM for {object_type} with parameters: {meta}") if meta is None: meta = {} raw_response = self.query_object(object_type, meta) - response = raw_response['sm:QueryResult']['sm:part'][1:] - - if type(response) == str: + if raw_response.get('sm:OperationFailure'): + logger.error(f"OPDM responded with error: {raw_response['sm:OperationFailure']['sm:part']}") response = [] - - logger.info(f"Number of responses: {len(response)} for query") + else: + # Extract metadata objects from response + response = raw_response['sm:QueryResult']['sm:part'][4:] + if isinstance(response, str): + response = [] + logger.info(f"Number of responses for query: {len(response)}") return response - def download_object(self, opdm_object, output_format='bytes', output_dir=None): + def download_object(self, opdm_object: dict): - if output_format == 'file': - for cimxml_file in opdm_object['opde:Component']: - file_id = cimxml_file['opdm:Profile']['opde:Id'] - file_name = cimxml_file['opdm:Profile']['pmd:fileName'] - file_path = os.path.join(output_dir, file_name) + model_meta = opdm_object + party = model_meta.get('pmd:modelPartReference', model_meta.get('pmd:TSO', '')) - logger.info(f"Downloading {file_name} with ID -> {file_id}") - response = self.get_content(file_id, return_payload=True) + for pos, model_part in enumerate(model_meta['opde:Component']): + model_part_meta = model_part['opdm:Profile'] + model_part_name = model_part_meta['pmd:fileName'] - with open(file_path, 'wb') as file_object: - message64_bytes = response['sm:GetContentResult']['sm:part'][1]['opdm:Profile']['opde:Content'].encode() - file_object.write(base64.b64decode(message64_bytes)) + # Maybe the file is all ready there (if OPDM subscription is enabled) + content_data = self.get_file(model_part_name) - logger.info(f"Saved to {file_path}") + # If file is not available on local client, lets request it from OPDM and download it + # More optimal, downloads whole model first (4 files) + if not content_data: + logger.warning("File not present on local client, requesting from OPDM service the whole Model") + content_meta = self.get_content(model_meta['opde:Id'], object_type="model") + content_data = self.get_file(model_part_name) - return None + # Less optimal, downloads each file separately + if not content_data: + logger.warning("File not present on local client, requesting from OPDM service the specific File") + content_meta = self.get_content(model_part_meta['opde:Id']) + content_data = self.get_file(model_part_name) - if output_format == 'bytes': - model_meta = opdm_object - party = model_meta.get('pmd:modelPartReference', model_meta.get('pmd:TSO', '')) + # If no data available set to None + if not content_data: + logger.error(f"{model_part_name} not available on webdav") + opdm_object['opde:Component'][pos]['opdm:Profile']["DATA"] = None - for pos, model_part in enumerate(model_meta['opde:Component']): - model_part_meta = model_part['opdm:Profile'] - model_part_name = model_part_meta['pmd:fileName'] + # Save data to metadata object + opdm_object['opde:Component'][pos]['opdm:Profile']["DATA"] = content_data - # Maybe the file is all ready there (if OPDM subscription is enabled) - content_data = self.get_file(model_part_name) + return opdm_object - # If file is not available on local client, lets request it from OPDM and download it + def download_object_to_filesystem(self, opdm_object: dict, output_dir: str | None = None): - # More optimal, downloads whole model first (4 files) - if not content_data: - logger.warning("File not present on local client, requesting from OPDM service the whole Model") - content_meta = self.get_content(model_meta['opde:Id'], object_type="model") - content_data = self.get_file(model_part_name) + for cimxml_file in opdm_object['opdm:OPDMObject']['opde:Component']: + file_id = cimxml_file['opdm:Profile']['opde:Id'] + file_name = cimxml_file['opdm:Profile']['pmd:fileName'] + file_path = os.path.join(output_dir, file_name) - # Less optimal, downloads each file separately - if not content_data: - logger.warning("File not present on local client, requesting from OPDM service the specific File") - content_meta = self.get_content(model_part_meta['opde:Id']) - content_data = self.get_file(model_part_name) + logger.info(f"Downloading {file_name} with ID: {file_id}") + response = self.get_content(file_id, return_payload=True) - # If no data available set to None - if not content_data: - logger.error(f"{model_part_name} not available on webdav") - opdm_object['opde:Component'][pos]['opdm:Profile']["DATA"] = None + with open(file_path, 'wb') as file_object: + message64_bytes = response['sm:GetContentResult']['sm:part'][4]['opdm:Profile']['opde:Content'].encode() + file_object.write(base64.b64decode(message64_bytes)) - # Save data to metadata object - opdm_object['opde:Component'][pos]['opdm:Profile']["DATA"] = content_data + logger.info(f"File saved to: {file_path}") - return opdm_object + return None def get_file(self, file_id): diff --git a/emf/common/integrations/rabbit.py b/emf/common/integrations/rabbit.py index 557f420c..7694156f 100644 --- a/emf/common/integrations/rabbit.py +++ b/emf/common/integrations/rabbit.py @@ -6,6 +6,7 @@ from typing import List from emf.common.config_parser import parse_app_properties from concurrent.futures import ThreadPoolExecutor, ProcessPoolExecutor + # from pika.adapters.asyncio_connection import AsyncioConnection # import asyncio @@ -93,7 +94,8 @@ def consume_start(self, queue: str, callback: object | None = None, auto_ack: bo # Set up consumer if not callback: - callback = lambda ch, method, properties, body: logger.info(f"Received message: {properties} (No callback processing)") + callback = lambda ch, method, properties, body: logger.info( + f"Received message: {properties} (No callback processing)") self.consume_channel.basic_consume( queue=queue, @@ -174,7 +176,7 @@ def __init__(self, port: int = int(RMQ_PORT), vhost: str = RMQ_VHOST, queue: str | None = None, - reply_to: str | None = None, + forward: str | None = None, username: str = RMQ_USERNAME, password: str = RMQ_PASSWORD, heartbeat: str | int = RMQ_HEARTBEAT_IN_SEC, @@ -183,14 +185,14 @@ def __init__(self, """Create a new instance of the consumer class, passing in the AMQP URL used to connect to RabbitMQ. - If 'reply_to' is provided messaged will be published to given queue name + If 'forward' is provided messaged will be published to given queue/exchange name """ self.message_handlers = message_handlers self.message_converter = message_converter self.should_reconnect = False self.was_consuming = False - self.reply_to = reply_to + self.forward = forward self._connection = None self._channel = None @@ -212,7 +214,8 @@ def __init__(self, port=self._port, virtual_host=self._vhost, credentials=pika.PlainCredentials(username, password), - heartbeat= None if RMQ_HEARTBEAT_IN_SEC == "0" else int(RMQ_HEARTBEAT_IN_SEC)) + heartbeat=None if RMQ_HEARTBEAT_IN_SEC == "0" else int( + RMQ_HEARTBEAT_IN_SEC)) def connect(self): """This method connects to RabbitMQ, returning the connection handle. @@ -223,7 +226,7 @@ def connect(self): """ logger.info(f"Connecting to {self._host}:{self._port} @ {self._vhost} as {self._username}") - + return pika.SelectConnection( parameters=self._connection_parameters, on_open_callback=self.on_connection_open, @@ -397,13 +400,20 @@ def _process_messages(self, basic_deliver, properties, body): # self.connection.close() # self.stop() - # Publish message to next exchange/queue if provided - if self.reply_to: - logger.info(f"Publishing message to exchange/queue: {self.reply_to}") - self._channel.basic_publish(exchange='', routing_key=self.reply_to, body=body, properties=properties) - + # Process message acknowledgment if ack: - self.acknowledge_message(basic_deliver.delivery_tag) + # Check if properties has some status flag set from handler + _success = properties.headers.get('success', True) + if _success: + # Publish message to next exchange/queue if provided and message successfully handled + if self.forward: + logger.info(f"Publishing message to exchange: {self.forward}") + self._channel.basic_publish(exchange=self.forward, routing_key="", body=body, properties=properties) + self.acknowledge_message(basic_deliver.delivery_tag) + logger.info("Message acknowledged") + else: + logger.warning(f"Task rejected due to success flag set by handler: {_success}") + self._channel.basic_reject(basic_deliver.delivery_tag, requeue=False) def on_message(self, _unused_channel, basic_deliver, properties, body): """Invoked by pika when a message is delivered from RabbitMQ. The @@ -417,7 +427,8 @@ def on_message(self, _unused_channel, basic_deliver, properties, body): :param pika.Spec.BasicProperties: properties :param bytes body: The message body """ - logger.info(f"Received message # {basic_deliver.delivery_tag} from {properties.app_id} meta: {properties.headers}") + logger.info( + f"Received message # {basic_deliver.delivery_tag} from {properties.app_id} meta: {properties.headers}") logger.debug(f"Message body: {body}") self._executor.submit(self._process_messages, basic_deliver, properties, body) @@ -494,12 +505,13 @@ class ReconnectingConsumer: """This is an example consumer that will reconnect if the nested RMQConsumer indicates that a reconnect is necessary. """ + def __init__(self, host: str = RMQ_SERVER, port: int = int(RMQ_PORT), vhost: str = RMQ_VHOST, queue: str | None = None, - reply_to: str | None = None, + forward: str | None = None, username: str = RMQ_USERNAME, password: str = RMQ_PASSWORD, message_handler: object | None = None, @@ -509,7 +521,7 @@ def __init__(self, self._port = port self._vhost = vhost self._queue = queue - self._reply_to = reply_to + self._forward = forward self._username = username self.__password = password self.message_handler = message_handler @@ -518,7 +530,7 @@ def __init__(self, port=self._port, vhost=self._vhost, queue=self._queue, - reply_to=self._reply_to, + forward=self._forward, username=self._username, password=self.__password, message_handlers=self.message_handler, @@ -557,6 +569,7 @@ def stop(self): if __name__ == '__main__': # Testing RMQ API import sys + logging.basicConfig(stream=sys.stdout, format="%(levelname) -10s %(asctime) -10s %(name) -35s %(funcName) -30s %(lineno) -5d: %(message)s", level=logging.INFO) diff --git a/emf/common/loadflow_tool/helper.py b/emf/common/loadflow_tool/helper.py deleted file mode 100644 index a94cf037..00000000 --- a/emf/common/loadflow_tool/helper.py +++ /dev/null @@ -1,644 +0,0 @@ -from zipfile import ZipFile, ZIP_DEFLATED -from uuid import uuid4 -from io import BytesIO -from inspect import ismethod -from typing import List -from datetime import datetime -import pandas as pd -import pypowsybl -import json -import logging -import pandas -import os -from lxml import etree -import triplets -import uuid -from aniso8601 import parse_datetime -import re - -logger = logging.getLogger(__name__) - -powsybl_default_export_settings = { - "iidm.export.cgmes.base-name": "", - "iidm.export.cgmes.cim-version": "", # 14, 16, 100 - "iidm.export.cgmes.export-boundary-power-flows": "true", - "iidm.export.cgmes.export-power-flows-for-switches": "true", - "iidm.export.cgmes.naming-strategy": "identity", # identity, cgmes, cgmes-fix-all-invalid-ids - "iidm.export.cgmes.profiles": "EQ,TP,SSH,SV", - "iidm.export.cgmes.boundary-EQ-identifier": "", - "iidm.export.cgmes.boundary-TP-identifier": "", - "iidm.export.cgmes.modeling-authority-set": "powsybl.org" -} - - -# TODO - Add comments and docstring -def package_for_pypowsybl(opdm_objects, return_zip: bool = False): - """ - Method to transform OPDM objects into sufficient format binary buffer or zip package - :param opdm_objects: list of OPDM objects - :param return_zip: flag to save OPDM objects as zip package in local directory - :return: binary buffer or zip package file name - """ - output_object = BytesIO() - if return_zip: - output_object = f"{uuid4()}.zip" - logging.info(f"Adding files to {output_object}") - - with ZipFile(output_object, "w") as global_zip: - for opdm_components in opdm_objects: - for instance in opdm_components['opde:Component']: - with ZipFile(BytesIO(instance['opdm:Profile']['DATA'])) as instance_zip: - for file_name in instance_zip.namelist(): - logging.info(f"Adding file: {file_name}") - global_zip.writestr(file_name, instance_zip.open(file_name).read()) - - return output_object - - -def save_opdm_objects(opdm_objects: list) -> list: - """ - Function save OPDM objects on to local filesystem - :param opdm_objects: list of OPDM objects - :return: list of exported files - """ - exported_files = [] - for opdm_components in opdm_objects: - for instance in opdm_components['opde:Component']: - file_name = instance['opdm:Profile']['pmd:fileName'] - logger.info(f'Saving - {file_name}') - with open(file_name, 'wb') as instance_zip: - instance_zip.write(instance['opdm:Profile']['DATA']) - exported_files.append(file_name) - - return exported_files - - -def clean_data_from_opdm_objects(opdm_objects: list) -> list: - for opdm_object in opdm_objects: - for component in opdm_object['opde:Component']: - component['opdm:Profile']['DATA'] = None - - return opdm_objects - - -def create_opdm_objects(models: list, metadata=None, key_profile="SV") -> list: - """ - Function to create OPDM object like structure in memory - input models, is a nested list [[EQ, SSH, TP, SV], [EQ, SSH, TP, SC]] where upper list will become opdm:Object and nested will be opdm:Profile - - :return: list of OPDM objects - """ - opdm_objects = [] - - for model in models: - opdm_object = {'opde:Component': []} - - for profile_instance in model: - - # Build profile instance metadata - opdm_profile = opdm_metadata_from_filename(profile_instance.name) - opdm_profile.update(opdm_metadata_from_rdfxml(get_xml_from_zip(profile_instance))) - opdm_profile['pmd:fileName'] = profile_instance.name - opdm_profile["pmd:content-reference"] = generate_OPDM_ContentReference_from_filename(profile_instance.name) - - # Check if key profile and add to main object metadata - if opdm_profile.get('pmd:cgmesProfile') == key_profile: - opdm_object.update(opdm_profile) - - # Add model type - IGM/CGM - # TODO develop logic to also define as CGM metadata object - opdm_object['opde:Object-Type'] = "IGM" - - # Add DATA - opdm_profile['DATA'] = profile_instance.getvalue() - - # Add component to main object - opdm_object['opde:Component'].append({'opdm:Profile': opdm_profile}) - - # If metadata provided, overwrite existing - if metadata: - opdm_object.update(metadata) - - opdm_objects.append(opdm_object) - - return opdm_objects - - -def attr_to_dict(instance: object, sanitize_to_strings: bool = False): - """ - Method to return class variables/attributes as dictionary - Example: LimitViolation(subject_id='e49a61d1-632a-11ec-8166-00505691de36', subject_name='', limit_type=HIGH_VOLTAGE, limit=450.0, limit_name='', acceptable_duration=2147483647, limit_reduction=1.0, value=555.6890952917897, side=ONE) - pypowsybl._pypowsybl.LimitViolation -> dict - :param instance: class instance - :param sanitize_to_strings: flag to convert attributes to string type - :return: dict - """ - - def convert_value(value): - if isinstance(value, datetime): - return value.isoformat() if sanitize_to_strings else value - elif isinstance(value, list): - return [convert_value(item) for item in value] - elif isinstance(value, dict): - return {k: convert_value(v) for k, v in value.items()} - elif hasattr(value, '__dict__'): - return attr_to_dict(value, sanitize_to_strings) - elif sanitize_to_strings: - return str(value) - return value - - attribs = [attr for attr in dir(instance) if (not callable(getattr(instance, attr)) and not attr.startswith("_"))] - result_dict = {attr_key: convert_value(getattr(instance, attr_key)) for attr_key in attribs} - - return result_dict - - -def parse_pypowsybl_report(report: str): - lines = report.replace('+', '').splitlines() - all_network_dicts = [] - - current_dict = None - base_indent = None - - for line in lines: - stripped_line = line.strip() - - # Identify "Network info" line and its indentation level - if "Network info" in stripped_line: - if current_dict is not None: - # Save the current dictionary if a new "Network info" block starts - all_network_dicts.append(current_dict) - - current_dict = {} - base_indent = len(line) - len(stripped_line) - continue - - if current_dict is not None: - # Calculate the current line's indentation level relative to "Network info" - current_indent = len(line) - len(line.lstrip()) - - # Check for the specific phrase "Network has x buses and y branches" - match = re.match(r"Network has (\d+) buses and (\d+) branches", stripped_line) - if match: - buses = int(match.group(1)) - branches = int(match.group(2)) - current_dict['buses'] = buses - current_dict['branches'] = branches - - # Process lines with key-value pairs after ':' - elif ':' in stripped_line: - dict_name, key_values = stripped_line.split(':', 1) - dict_name = dict_name.strip() - key_values = key_values.strip() - - # Parse key-value pairs - if '=' in key_values: - current_dict[dict_name] = {} - for pair in key_values.split(','): - key, value = map(str.strip, pair.split('=')) - current_dict[dict_name][key] = value - else: - # Handle plain strings after ':' - current_dict[dict_name] = key_values - - else: - # Stop processing this block if indentation level is not greater than base_indent - if current_indent <= base_indent and current_dict: - all_network_dicts.append(current_dict) - current_dict = None - - # Append the last dictionary if it exists - if current_dict is not None and current_dict: - all_network_dicts.append(current_dict) - - # Filter out empty dicts - result = [n for n in all_network_dicts if n] - - return result - - -def get_network_elements(network: pypowsybl.network, - element_type: pypowsybl.network.ElementType, - all_attributes: bool = True, - attributes: List[str] = None, - **kwargs - ): - - _voltage_levels = network.get_voltage_levels(all_attributes=True).rename(columns={"name": "voltage_level_name"}) - _substations = network.get_substations(all_attributes=True).rename(columns={"name": "substation_name"}) - - elements = network.get_elements(element_type=element_type, all_attributes=all_attributes, attributes=attributes, **kwargs) - elements = elements.merge(_voltage_levels, left_on='voltage_level_id', right_index=True, suffixes=(None, '_voltage_level')) - elements = elements.merge(_substations, left_on='substation_id', right_index=True, suffixes=(None, '_substation')) - - # Need to ensure that column 'isHvdc' is present if DANGLING_LINE type is requested - if element_type is pypowsybl.network.ElementType.DANGLING_LINE: - if 'isHvdc' not in elements.columns: - elements['isHvdc'] = '' - - return elements - - -def get_slack_generators(network: pypowsybl.network): - - slack_terminals = network.get_extension('slackTerminal') - slack_generators = get_network_elements(network=network, - element_type=pypowsybl.network.ElementType.GENERATOR, - all_attributes=True, - id=slack_terminals['element_id']) - - return slack_generators - - -def get_connected_components_data(network: pypowsybl.network, - bus_count_threshold: int | None = None, - country_col_name: str = 'country'): - buses = get_network_elements(network, pypowsybl.network.ElementType.BUS) - data = buses.groupby('connected_component').agg(countries=(country_col_name, lambda x: list(x.unique())), - bus_count=('name', 'size')) - if bus_count_threshold: - data = data[data.bus_count > bus_count_threshold] - - return data.to_dict('index') - - -def load_model(opdm_objects: List[dict], parameters: dict = None, skip_default_parameters: bool = False): - """ - Loads given list of models (opdm_objects) into pypowsybl using internal (known good) default_parameters - Additional parameters can be specified as a dict in field parameters which will overwrite the default ones if keys - are matching - :param opdm_objects: list of dictionaries following the opdm model format - :param parameters: dictionary of desired parameters for loading models to pypowsybl - :param skip_default_parameters: skip the default parameters - """ - default_parameters = {"iidm.import.cgmes.import-node-breaker-as-bus-breaker": 'true'} - if not skip_default_parameters: - if not parameters: - parameters = default_parameters - else: - # Give a priority to parameters given from outside - parameters = {**default_parameters, **parameters} - - import_report = pypowsybl.report.Reporter() - network = pypowsybl.network.load_from_binary_buffer( - buffer=package_for_pypowsybl(opdm_objects), - reporter=import_report, - parameters=parameters - # parameters={ - # "iidm.import.cgmes.store-cgmes-model-as-network-extension": 'true', - # "iidm.import.cgmes.create-active-power-control-extension": 'true', - # "iidm.import.cgmes.post-processors": ["EntsoeCategory"]} - ) - - logger.info(f"Loaded: {network}") - logger.debug(f"{import_report}") - - return network - - -def opdmprofile_to_bytes(opdm_profile): - data = BytesIO(opdm_profile['opdm:Profile']['DATA']) - data.name = opdm_profile['opdm:Profile']['pmd:fileName'] - return data - - -def load_opdm_data(opdm_objects, profile=None): - if profile: - return pandas.read_RDF([opdmprofile_to_bytes(instance) for model in opdm_objects for instance in model['opde:Component'] if instance['opdm:Profile']['pmd:cgmesProfile'] == profile]) - return pandas.read_RDF([opdmprofile_to_bytes(instance) for model in opdm_objects for instance in model['opde:Component']]) - - -def get_opdm_data_from_models(model_data: list | pandas.DataFrame): - """ - Check if input is already parsed to triplets. Do it otherwise - :param model_data: input models - :return triplets - """ - if not isinstance(model_data, pandas.DataFrame): - model_data = load_opdm_data(model_data) - return model_data - - -def filename_from_opdm_metadata(metadata): - - model_part = metadata.get('pmd:modelPartReference', None) - - if model_part: - model_authority = f"{metadata['pmd:mergingEntity']}-{metadata['pmd:mergingArea']}-{model_part}" - - else: - model_authority = f"{metadata['pmd:mergingEntity']}-{metadata['pmd:mergingArea']}" - - file_name = f"{metadata['pmd:validFrom']}_{metadata['pmd:timeHorizon']}_{model_authority}_{metadata['pmd:cgmesProfile']}_{metadata['pmd:versionNumber']}" - file_name = ".".join([file_name, metadata["file_type"]]) - - return file_name - - -def opdm_metadata_from_filename(file_name: str, meta_separator: str = "_"): - """ - Parse OPDM metadata from a filename string into a dictionary. - - Args: - :param file_name: (str): The filename containing metadata separated by underscores - and a file extension - :param meta_separator: (str): How the elements are seperated in the string, usually by "_" - - Returns: - dict: Dictionary containing parsed metadata with OPDM-specific keys. - - Raises: - AssertionError: If file_name is not a string. - ValueError: If metadata parsing fails due to incorrect format. - """ - # Constants for opdm metadata keys - VALID_FROM = 'pmd:validFrom' - TIME_HORIZON = 'pmd:timeHorizon' - CGMES_PROFILE = 'pmd:cgmesProfile' - VERSION_NUMBER = 'pmd:versionNumber' - MERGING_ENTITY = 'pmd:mergingEntity' - MERGING_AREA = 'pmd:mergingArea' - MODEL_PART = 'pmd:modelPartReference' - TSO = "pmd:TSO" - SOURCING_ACTOR = "pmd:sourcingActor" - - # Constant for file type - FILE_TYPE = 'file_type' - - # Input validation - assert isinstance(file_name, str), "file_name must be a string" - - metadata = {} # Meta container - - # Split filename into name and extension - file_name, metadata[FILE_TYPE] = file_name.split(".") - meta_list = file_name.split(meta_separator) - - # Parse main metadata components - if len(meta_list) == 4: - metadata[VALID_FROM], model_authority, metadata[CGMES_PROFILE], metadata[VERSION_NUMBER] = meta_list - metadata[TIME_HORIZON] = "" - elif len(meta_list) == 5: - metadata[VALID_FROM], metadata[TIME_HORIZON], model_authority, metadata[CGMES_PROFILE], metadata[VERSION_NUMBER] = meta_list - else: - logger.warning(f"Parsing error, number of allowed meta in filename is 4 or 5 separated by {meta_separator} -> {file_name}") - return metadata - - # Parse model authority components - model_authority_list = model_authority.split("-") - - if len(model_authority_list) == 1: - metadata[MODEL_PART] = model_authority - elif len(model_authority_list) == 2: - metadata[MERGING_ENTITY], metadata[MERGING_AREA] = model_authority_list - elif len(model_authority_list) == 3: - metadata[MERGING_ENTITY], metadata[MERGING_AREA], metadata[MODEL_PART] = model_authority_list - else: - logger.error(f"Parsing error {model_authority}") - - # Add aliases - if metadata.get(MODEL_PART): - metadata[TSO] = metadata[MODEL_PART] - metadata[SOURCING_ACTOR] = metadata[MODEL_PART] - - return metadata - - -def metadata_from_rdfxml(parsed_xml: etree._ElementTree): - """Parse model metadata form xml, returns a dictionary with metadata""" - - assert isinstance(parsed_xml, etree._ElementTree) - - rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" - - header = parsed_xml.find("{*}FullModel") # Verion update proof, as long as element name remains the same - meta_elements = header.getchildren() - - # Add model ID from FullModel@about - metadata = {"Model.mRID": header.attrib.get(f'{{{rdf}}}about').split(":")[-1]} - - # Add all other metadata - for element in meta_elements: - - key = element.tag.split("}")[1] - - value = element.text if element.text else element.attrib.get(f"{{{rdf}}}resource") - - if existing_value := metadata.get(key): - value = existing_value + [value] if isinstance(existing_value, list) else [existing_value, value] - - metadata[key] = value - - return metadata - - -def opdm_metadata_from_rdfxml(parsed_xml: etree._ElementTree): - - opdm_metadata_map = { - - "Model.mRID": ["opde:Id", "pmd:modelid", "pmd:fullModel_ID"], - "Model.scenarioTime": ["pmd:scenarioDate"], - "Model.created": ["pmd:creationDate"], - "Model.description": ["pmd:description"], - "Model.version": ["pmd:version"], - "Model.DependentOn": ["opde:DependsOn"], - "Model.profile": ["pmd:modelProfile"], - "Model.modelingAuthoritySet": ["pmd:modelingAuthoritySet"], - - } - - metadata = {} - - for key, value in metadata_from_rdfxml(parsed_xml).items(): - if opdm_keys := opdm_metadata_map.get(key): - for opdm_key in opdm_keys: - metadata[opdm_key] = value - - return metadata - - -# TODO -# "pmd:fileName": "20250317T2230Z_14_D7_SSH_011.zip", -# "pmd:content-reference": "CGMES/14/D7/20250317/223000/SSH/20250317T2230Z_14_D7_SSH_011.zip", -# "pmd:Content-Type": "CGMES", - - -def get_xml_from_zip(zip_file_path): - - with ZipFile(zip_file_path, 'r') as zipfile_object: - xml_file_name = zipfile_object.namelist()[0] - file_bytes = zipfile_object.read(xml_file_name) - # Convert bytes to file-like object for parsing - xml_tree_object = etree.parse(BytesIO(file_bytes)) - - return xml_tree_object - - -def zip_xml(xml_file_object): - - xml_file_name = xml_file_object.name - xml_file_extension = xml_file_name.split(".")[-1] - zip_file_name = xml_file_name.replace(f".{xml_file_extension}", ".zip") - - zip_file_object = BytesIO() - zip_file_object.name = zip_file_name - - # Create and save ZIP - with ZipFile(zip_file_object, 'w', ZIP_DEFLATED) as zipfile: - zipfile.writestr(xml_file_name, xml_file_object.getvalue()) - - return zip_file_object - - -def get_metadata_from_filename(file_name): - - # Separators - file_type_separator = "." - meta_separator = "_" - entity_and_domain_separator = "-" - - logger.debug(file_name) - file_metadata = {} - file_name, file_type = file_name.split(file_type_separator) - - # Parse file metadata - file_meta_list = file_name.split(meta_separator) - - # Naming before QoDC 2.1, where EQ might not have processType - if len(file_meta_list) == 4: - - file_metadata["Model.scenarioTime"],\ - file_metadata["Model.modelingEntity"],\ - file_metadata["Model.messageType"],\ - file_metadata["Model.version"] = file_meta_list - file_metadata["Model.processType"] = "" - - logger.warning("Only 4 meta elements found, expecting 5, setting Model.processType to empty string") - - # Naming after QoDC 2.1, always 5 positions - elif len(file_meta_list) == 5: - - file_metadata["Model.scenarioTime"],\ - file_metadata["Model.processType"],\ - file_metadata["Model.modelingEntity"],\ - file_metadata["Model.messageType"],\ - file_metadata["Model.version"] = file_meta_list - - else: - logger.error("Non CGMES file {}".format(file_name)) - - if file_metadata.get("Model.modelingEntity"): - - entity_and_area_list = file_metadata["Model.modelingEntity"].split(entity_and_domain_separator) - - if len(entity_and_area_list) == 1: - file_metadata["Model.mergingEntity"],\ - file_metadata["Model.domain"] = "", "" # Set empty string for both - file_metadata["Model.forEntity"] = entity_and_area_list[0] - - if len(entity_and_area_list) == 2: - file_metadata["Model.mergingEntity"],\ - file_metadata["Model.domain"] = entity_and_area_list - file_metadata["Model.forEntity"] = "" - - if len(entity_and_area_list) == 3: - file_metadata["Model.mergingEntity"],\ - file_metadata["Model.domain"],\ - file_metadata["Model.forEntity"] = entity_and_area_list - - return file_metadata - - -def generate_OPDM_ContentReference_from_filename(file_name, opdm_object_type="CGMES"): - """ - Generates the file path based on the given parameters and metadata extracted from the filename. - - Parameters: - file_name (str): The name of the file from which metadata will be extracted. - opdm_object_type (str): The type of OPDM object, defaults to "CGMES". - - Returns: - str: Generated file path using the provided template and extracted metadata. - - Example: - >>> file_name = "example_filename" - >>> opdm_object_type = "CGMES" - >>> generate_OPDM_ContentReference_from_filename(file_name, opdm_object_type) - 'CGMES/processType/modelingEntity/20240529/123000/messageType/example_filename' - """ - template = "{opdm_object_type}/{processType}/{modelingEntity}/{scenarioTime:%Y%m%d}/{scenarioTime:%H%M00}/{messageType}/{file_name}" - - meta = {key.split(".")[-1]: value for key, value in get_metadata_from_filename(file_name).items()} - meta["scenarioTime"] = parse_datetime(meta["scenarioTime"]) - meta["file_name"] = file_name - meta["opdm_object_type"] = opdm_object_type - - return template.format(**meta) - - -def export_model(network: pypowsybl.network, opdm_object_meta, profiles=None): - - if profiles: - profiles = ",".join([str(profile) for profile in profiles]) - else: - profiles = "SV,SSH,TP,EQ" - - file_base_name = filename_from_opdm_metadata(opdm_object_meta).split(".xml")[0] - - bytes_object = network.save_to_binary_buffer( - format="CGMES", - parameters={ - "iidm.export.cgmes.modeling-authority-set": opdm_object_meta['pmd:modelingAuthoritySet'], - "iidm.export.cgmes.base-name": file_base_name, - "iidm.export.cgmes.profiles": profiles, - # "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids", # identity, cgmes, cgmes-fix-all-invalid-ids - "iidm.export.cgmes.export-sv-injections-for-slacks": "False", - }) - - bytes_object.name = f"{file_base_name}_{uuid.uuid4()}.zip" - - return bytes_object - - -def get_model_outages(network: pypowsybl.network): - - # Get network elements - lines = network.get_elements(element_type=pypowsybl.network.ElementType.LINE, all_attributes=True).reset_index(names=['grid_id']) - _voltage_levels = network.get_voltage_levels(all_attributes=True).rename(columns={"name": "voltage_level_name"}) - _substations = network.get_substations(all_attributes=True).rename(columns={"name": "substation_name"}) - lines = lines.merge(_voltage_levels, left_on='voltage_level1_id', right_index=True, suffixes=(None, '_voltage_level')) - lines = lines.merge(_substations, left_on='substation_id', right_index=True, suffixes=(None, '_substation')) - lines['element_type'] = 'LINE' - - dlines = get_network_elements(network, pypowsybl.network.ElementType.DANGLING_LINE).reset_index(names=['grid_id']) - dlines['element_type'] = 'DANGLING_LINE' - - gens = get_network_elements(network, pypowsybl.network.ElementType.GENERATOR).reset_index(names=['grid_id']) - gens['element_type'] = 'GENERATOR' - - # Filter out disconnected elements - disconnected_lines = lines[(lines['connected1'] == False) | (lines['connected2'] == False)] - disconnected_dlines = dlines[dlines['connected'] == False] - disconnected_gens = gens[gens['connected'] == False] - - # Filter out only 330kv and above - disconnected_elements = pd.concat([disconnected_lines, disconnected_dlines, disconnected_gens]) - disconnected_elements = disconnected_elements[disconnected_elements['nominal_v'] >= 330] - - return disconnected_elements.to_dict('records') - - -if __name__ == "__main__": - - example_file_path = r"C:\Users\kristjan.vilgo\OneDrive - Elering AS\Documents\GitHub\igm_publication\20250314T2330Z_1D_ELERING_012.zip" - - # Find all XML regardless the zip folder depth - rdfxml_files = triplets.rdf_parser.find_all_xml([example_file_path]) - - # Repackage to form zip(xml) - rdfzip_files = [] - for xml in rdfxml_files: - rdfzip_files.append(zip_xml(xml)) - - # Create OPDM objects - opdm_object = create_opdm_objects([rdfzip_files]) diff --git a/emf/common/loadflow_tool/load_files_general.py b/emf/common/loadflow_tool/load_files_general.py index c3179f19..a7e7932b 100644 --- a/emf/common/loadflow_tool/load_files_general.py +++ b/emf/common/loadflow_tool/load_files_general.py @@ -1,7 +1,8 @@ import os import logging -from emf.common.loadflow_tool.helper import opdm_metadata_from_filename + +from emf.common.helpers.opdm_objects import get_metadata_from_file_name OPDE_COMPONENT_KEYWORD = 'opde:Component' OPDM_PROFILE_KEYWORD = 'opdm:Profile' @@ -69,7 +70,7 @@ def get_meta_from_filename(file_name: str): if key in fixed_file_name: fixed_file_name = fixed_file_name.replace(key, BOUNDARY_FILE_TYPE_FIX[key]) # meta_data = get_metadata_from_filename(fixed_file_name) - meta_data = opdm_metadata_from_filename(fixed_file_name) + meta_data = get_metadata_from_file_name(fixed_file_name) # Revert back cases where there is a '-' in TSO's name like ENTSO-E for case in SPECIAL_TSO_NAME: if case in fixed_file_name: diff --git a/emf/model_merger/legacy.py b/emf/model_merger/legacy.py new file mode 100644 index 00000000..8e8a9797 --- /dev/null +++ b/emf/model_merger/legacy.py @@ -0,0 +1,130 @@ +import pandas as pd +import triplets + + +def configure_paired_boundarypoint_injections(data): + # TODO [LEGACY] + """Where there are paired boundary points, equivalent injections need to be modified + Set P and Q to 0 - so that no additional consumption or production is on tie line + Set voltage control off - so that no additional consumption or production is on tie line + Set terminal to connected - to be sure we have paired connected injections at boundary point + """ + boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] + boundary_points = boundary_points.merge(data.type_tableview("Terminal").reset_index(), + left_on="ID", + right_on="Terminal.ConnectivityNode", + suffixes=('_ConnectivityNode', '_Terminal')) + injections = data.type_tableview('EquivalentInjection').reset_index().merge(boundary_points, + left_on="ID", + right_on='Terminal.ConductingEquipment', + suffixes=('_ConnectivityNode', '')) + + # Get paired injections at boundary points + paired_injections = injections.groupby("Terminal.ConnectivityNode").filter(lambda x: len(x) == 2) + + # Set terminal status + updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) + updated_terminal_status["KEY"] = "ACDCTerminal.connected" + updated_terminal_status["VALUE"] = "true" + + # Set Regulation off + updated_regulation_status = paired_injections[["ID"]].copy() + updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" + updated_regulation_status["VALUE"] = "false" + + # Set P to 0 + updated_p_value = paired_injections[["ID"]].copy() + updated_p_value["KEY"] = "EquivalentInjection.p" + updated_p_value["VALUE"] = 0 + + # Set Q to 0 + updated_q_value = paired_injections[["ID"]].copy() + updated_q_value["KEY"] = "EquivalentInjection.q" + updated_q_value["VALUE"] = 0 + + return data.update_triplet_from_triplet(pd.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) + + +def configure_paired_boundarypoint_injections_by_nodes(data): + # TODO [LEGACY] + """Where there are paired boundary points, equivalent injections need to be modified + Set P and Q to 0 - so that no additional consumption or production is on tie line + Set voltage control off - so that no additional consumption or production is on tie line + Set terminal to connected - to be sure we have paired connected injections at boundary point + + TODO NOTE THAT THIS IS COPY FROM 'configure_paired_boundarypoint_injections' + In some models terminals are missing references to ConnectivityNodes + """ + connectivity_boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] + topological_boundary_points = data.query("KEY == 'TopologicalNode.boundaryPoint' and VALUE == 'true'")[["ID"]] + try: + terminals = data.type_tableview("Terminal").reset_index()[['ID', + 'Terminal.ConductingEquipment', + 'Terminal.ConnectivityNode', + 'Terminal.TopologicalNode']] + except KeyError: + terminals = data.type_tableview("Terminal").reset_index()[['ID', + 'Terminal.ConductingEquipment', + 'Terminal.TopologicalNode']] + injections = data.type_tableview('EquivalentInjection').reset_index()[['ID', + # 'EquivalentInjection.p', + # 'EquivalentInjection.q', + # 'EquivalentInjection.regulationStatus' + ]] + topological_boundary_points = topological_boundary_points.merge(terminals, + left_on="ID", + right_on="Terminal.TopologicalNode", + suffixes=('_TopologicalNode', '_Terminal')) + topological_injections = injections.merge(topological_boundary_points, + left_on="ID", + right_on='Terminal.ConductingEquipment', + suffixes=('_ConnectivityNode', '')) + paired_topological_injections = (topological_injections.groupby("Terminal.TopologicalNode") + .filter(lambda x: len(x) == 2)) + paired_injections = paired_topological_injections + if 'Terminal.ConnectivityNode' in terminals: + connectivity_boundary_points = connectivity_boundary_points.merge(terminals, + left_on="ID", + right_on="Terminal.ConnectivityNode", + suffixes=('_ConnectivityNode', '_Terminal')) + connectivity_injections = injections.merge(connectivity_boundary_points, + left_on="ID", + right_on='Terminal.ConductingEquipment', + suffixes=('_TopologicalNode', '')) + + paired_connectivity_injections = (connectivity_injections.groupby("Terminal.ConnectivityNode") + .filter(lambda x: len(x) == 2)) + merged_injections = paired_connectivity_injections.merge(paired_topological_injections, + on='ID', + how='outer', + indicator=True, + suffixes=('_CN', '_TN')) + only_connectivity_injections = merged_injections[merged_injections['_merge'] == 'left_only'] + only_topological_injections = merged_injections[merged_injections['_merge'] == 'right_only'] + if len(only_connectivity_injections.index) != 0 or len(only_topological_injections.index) == 0: + paired_injections = paired_connectivity_injections + else: + logger.warning(f"Mismatch of finding paired injections from topological nodes and connectivity nodes") + else: + logger.warning(f"Terminals do not contain Connectivity nodes") + # Set terminal status + updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) + updated_terminal_status["KEY"] = "ACDCTerminal.connected" + updated_terminal_status["VALUE"] = "true" + + # Set Regulation off + updated_regulation_status = paired_injections[["ID"]].copy() + updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" + updated_regulation_status["VALUE"] = "false" + + # Set P to 0 + updated_p_value = paired_injections[["ID"]].copy() + updated_p_value["KEY"] = "EquivalentInjection.p" + updated_p_value["VALUE"] = 0 + + # Set Q to 0 + updated_q_value = paired_injections[["ID"]].copy() + updated_q_value["KEY"] = "EquivalentInjection.q" + updated_q_value["VALUE"] = 0 + + return data.update_triplet_from_triplet(pd.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) \ No newline at end of file diff --git a/emf/model_merger/merge_functions.py b/emf/model_merger/merge_functions.py index ac9ee348..5a65dee1 100644 --- a/emf/model_merger/merge_functions.py +++ b/emf/model_merger/merge_functions.py @@ -1,26 +1,25 @@ -import zipfile +import json import math -from io import BytesIO +from xml.sax.expatreader import version +import pandas as pd import pypowsybl import logging -import json import sys -from decimal import Decimal import datetime import triplets -import pandas import uuid -from uuid import uuid4 import config -from emf.common.loadflow_tool.helper import load_model, load_opdm_data, filename_from_opdm_metadata, attr_to_dict, \ - export_model, get_network_elements, get_opdm_data_from_models -from emf.common.loadflow_tool import loadflow_settings -from aniso8601 import parse_datetime -from emf.common.time_helper import parse_datetime as helper_parse_datetime - +from emf.common.config_parser import parse_app_properties +from emf.common.integrations import elastic +from emf.model_merger import temporary +from emf.common.helpers.time import parse_datetime +from emf.common.helpers.loadflow import get_model_outages, get_network_elements +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, filename_from_opdm_metadata logger = logging.getLogger(__name__) -SV_INJECTION_LIMIT = 0.1 + +parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.post_processing) + def is_valid_uuid(uuid_value): """ @@ -35,57 +34,76 @@ def is_valid_uuid(uuid_value): except ValueError: return False -def run_lf(merged_model, loadflow_settings=loadflow_settings.CGM_DEFAULT): - - # report = pypowsybl.report.Reporter() - result = pypowsybl.loadflow.run_ac(network=merged_model.network, - parameters=loadflow_settings, - # reporter=loadflow_report, - ) - - result_dict = [attr_to_dict(island) for island in result] - # Modify all nested objects to native data types - for island in result_dict: - island['status'] = island['status'].name - # Extract only first slack bus from internal pypowsybl object - slack_bus_results = island.pop('slack_bus_results') - if slack_bus_results: - island['slack_bus_id'] = getattr(slack_bus_results[0], 'id', 'undefined') - island['active_power_mismatch'] = getattr(slack_bus_results[0], 'active_power_mismatch', float()) - else: - island['slack_bus_id'] = 'undefined' - island['active_power_mismatch'] = float() - # merged_model.loadflow = json.loads(loadflow_report.to_json()) - # merged_model.loadflow = str(loadflow_report) - merged_model.loadflow = [island for island in result_dict if island['reference_bus_id']] - merged_model.loadflow_status = result[0].status.name # store main island loadflow status +def export_merged_model(network: pypowsybl.network, + opdm_object_meta: dict, + profiles: list[str] | None = None, + cgm_convention: bool = True, + ): - return merged_model + # Define which profiles to export + if profiles: + profiles = ",".join(profiles) + else: + profiles = "SV,SSH,TP,EQ" + + # Define whether export using CGM official structure + cgm_export_flag = "False" + if cgm_convention: + cgm_export_flag = "True" + + # Define base name for exported files + file_base_name = filename_from_opdm_metadata(metadata=opdm_object_meta) + + # Define CGMES export parameters + parameters = { + "iidm.export.cgmes.modeling-authority-set": opdm_object_meta['pmd:modelingAuthoritySet'], + "iidm.export.cgmes.base-name": file_base_name, + "iidm.export.cgmes.profiles": profiles, + # For missing instances like "SupplyStation" + "iidm.export.cgmes.topology-kind": 'NODE_BREAKER', + # cgmes-fix-all-invalid-ids fixes non-standard uuid's. Can cause danglingReference errors + # "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids", # identity, cgmes, cgmes-fix-all-invalid-ids + "iidm.export.cgmes.export-sv-injections-for-slacks": "False", + # False sets all boundary flows to zero causing Kirchhoff 1st law and SvPowerFlowBranchInstances2 errors + # "iidm.export.cgmes.export-boundary-power-flows": "False", + "iidm.export.cgmes.cgm_export": cgm_export_flag, + } + + # Export to bytes object + bytes_object = network.save_to_binary_buffer(format="CGMES", parameters=parameters) + bytes_object.name = f"{file_base_name}_{uuid.uuid4()}.zip" + logger.info(f"Exported merged model to {bytes_object.name}") + # TODO set correct naming of exported files + # a = triplets.rdf_parser.find_all_xml([bytes_object]) -def create_opdm_object_meta(object_id, - time_horizon, - merging_entity, - merging_area, - scenario_date, - mas, - version, - profile, - content_type="CGMES", - file_type="xml" - ): + return bytes_object + + +def create_merged_model_opdm_object(object_id: str, + time_horizon: str, + merging_entity: str, + merging_area: str, + scenario_date: datetime.datetime, + mas: str, + version: str = "001", + profile: str = "SV", + content_type: str = "CGMES", + ): opdm_object_meta = { + 'opde:Object-Type': 'CGM', 'pmd:fullModel_ID': object_id, - 'pmd:creationDate': f"{datetime.datetime.utcnow():%Y-%m-%dT%H:%M:%S.%fZ}", + 'pmd:creationDate': f"{datetime.datetime.now(datetime.UTC):%Y-%m-%dT%H:%M:%S.%fZ}", 'pmd:timeHorizon': time_horizon, 'pmd:cgmesProfile': profile, 'pmd:contentType': content_type, - 'pmd:modelPartReference': '', + 'pmd:modelPartReference': "-".join([merging_entity, merging_area]), 'pmd:mergingEntity': merging_entity, - 'pmd:mergingArea': merging_area, + 'pmd:Area': merging_area, 'pmd:validFrom': f"{parse_datetime(scenario_date):%Y%m%dT%H%MZ}", 'pmd:modelingAuthoritySet': mas, + "pmd:isFullModel": "true", 'pmd:scenarioDate': f"{parse_datetime(scenario_date):%Y-%m-%dT%H:%M:00Z}", 'pmd:modelid': object_id, 'pmd:description': f""" @@ -95,105 +113,77 @@ def create_opdm_object_meta(object_id, Model: Simplification of reality for given need. """, 'pmd:versionNumber': f"{int(version):03d}", - 'file_type': file_type } return opdm_object_meta -def update_FullModel_from_OpdmObject(data, opdm_object): +def update_header_from_opdm_object(data: pd.DataFrame, opdm_object: dict): return triplets.cgmes_tools.update_FullModel_from_dict(data, metadata={ "Model.version": f"{int(opdm_object['pmd:versionNumber']):03d}", "Model.created": f"{parse_datetime(opdm_object['pmd:creationDate']):%Y-%m-%dT%H:%M:%S.%fZ}", "Model.mergingEntity": opdm_object['pmd:mergingEntity'], - "Model.domain": opdm_object['pmd:mergingArea'], + "Model.domain": opdm_object['pmd:Area'], "Model.scenarioTime": f"{parse_datetime(opdm_object['pmd:scenarioDate']):%Y-%m-%dT%H:%M:00Z}", "Model.description": opdm_object['pmd:description'], - "Model.processType": opdm_object['pmd:timeHorizon'] + "Model.processType": opdm_object['pmd:timeHorizon'], }) -def revert_ids_back(exported_model, triplets_data, revert_ids: bool = True): - """ - As pypowsybl creates its own unique uuids for the cases when the originals do not match the criteria then this - takes the naming_strategy.csv provided by the pypowsybl and reverts those ids back if it is applicable - :param exported_model: binary object from pypowsybl - :param triplets_data: profile(s) converted to triplets - :param revert_ids: True = fix, False=report only - :return (updated) triplets data - """ - - contents = zipfile.ZipFile(exported_model) - naming_strategy = pandas.DataFrame() - - for file_name in contents.namelist(): - if 'naming_strategy' in file_name: - naming_strategy = pandas.read_csv(filepath_or_buffer=BytesIO(contents.read(file_name)), sep=';') - break - if not naming_strategy.empty: - existing_values = triplets_data.merge(naming_strategy, left_on='VALUE', right_on='CgmesUuid') - existing_values = existing_values[existing_values['IidmId'] != 'unknown'] - if not existing_values.empty: - if not revert_ids: - logger.error(f"Found {len(existing_values.index)} changed ids, consider dangling reference errors") - return triplets_data - logger.warning(f"Mapping {len(existing_values.index)} ids back") - existing_values['VALUE'] = existing_values['IidmId'] - new_existing_values = existing_values[['ID', 'KEY', 'VALUE', 'INSTANCE_ID']] - triplets_data = triplets.rdf_parser.update_triplet_from_triplet(triplets_data, new_existing_values) - return triplets_data - - - -def create_sv_and_updated_ssh(merged_model, original_models, models_as_triplets, scenario_date, time_horizon, version, merging_area, merging_entity, mas): - +def update_merged_model_sv(sv_data: bytes, opdm_object_meta: dict): - ### SV ### - # Set Metadata - SV_ID = merged_model.network_meta['id'].split("uuid:")[-1] + # Load SV profile data + sv_data = pd.read_RDF([sv_data]) - opdm_object_meta = create_opdm_object_meta(SV_ID, - time_horizon, - merging_entity, - merging_area, - scenario_date, - mas, - version, - profile="SV") + # Update rdfxml header from opdm object metadata + sv_data = update_header_from_opdm_object(data=sv_data, opdm_object=opdm_object_meta) - exported_model = export_model(merged_model.network, opdm_object_meta, ["SV"]) - logger.info(f"Exporting merged model to {exported_model.name}") + # Update file name at 'label' key + sv_data.set_VALUE_at_KEY(key='label', value=filename_from_opdm_metadata(opdm_object_meta, file_type="xml")) - # Load SV data - sv_data = pandas.read_RDF([exported_model]) - - # Fix naming - sv_data = revert_ids_back(exported_model=exported_model, triplets_data=sv_data) - - # Update - sv_data.set_VALUE_at_KEY(key='label', value=filename_from_opdm_metadata(opdm_object_meta)) sv_data = triplets.cgmes_tools.update_FullModel_from_filename(sv_data) - # Update metadata - sv_data = update_FullModel_from_OpdmObject(sv_data, opdm_object_meta) - - # Update filename - sv_data = triplets.cgmes_tools.update_filename_from_FullModel(sv_data) - - # Check and fix SV id + # Check and fix SV id if necessary updated_sv_id_map = {} for old_id in sv_data.query("KEY == 'Type' and VALUE == 'FullModel'").ID.unique(): if not is_valid_uuid(old_id): - new_id = str(uuid4()) + new_id = str(uuid.uuid4()) updated_sv_id_map[old_id] = new_id - logger.info(f"SV id: {old_id} is not valid. Assigning {new_id}") + logger.warning(f"SV profile id {old_id} is not valid, assigning: {new_id}") sv_data = sv_data.replace(updated_sv_id_map) + return sv_data + + +def load_ssh(input_data: pd.DataFrame | list): + """ + Loads in ssh profiles from list of profiles or takes the slice from dataframe + :param input_data: list of profiles or dataframe + :return dataframe of ssh data + """ + if not isinstance(input_data, pd.DataFrame): + ssh_data = load_opdm_objects_to_triplets(input_data, "SSH") + else: + ssh_files = (input_data[(input_data['KEY'] == 'label') & + (input_data['VALUE'].str.upper().str.contains('SSH'))][['INSTANCE_ID']] + .drop_duplicates()) + ssh_data = input_data.merge(ssh_files, on='INSTANCE_ID') + ssh_data = triplets.cgmes_tools.update_FullModel_from_filename(ssh_data) + return ssh_data + + +def create_updated_ssh(models_as_triplets: pd.DataFrame | list, + sv_data: pd.DataFrame, + opdm_object_meta: dict, + input_models: list = None, + ): + # TODO rewrite to use pypowsybl exported SSH + ### SSH ## # Load original SSH data to created updated SSH - ssh_data = load_opdm_data(original_models, "SSH") - ssh_data = triplets.cgmes_tools.update_FullModel_from_filename(ssh_data) + ssh_file_data = input_models or models_as_triplets + ssh_data = load_ssh(ssh_file_data) # Update SSH Model.scenarioTime ssh_data.set_VALUE_at_KEY('Model.scenarioTime', opdm_object_meta['pmd:scenarioDate']) @@ -262,7 +252,7 @@ def create_sv_and_updated_ssh(merged_model, original_models, models_as_triplets, # Generate new UUID for updated SSH updated_ssh_id_map = {} for OLD_ID in ssh_data.query("KEY == 'Type' and VALUE == 'FullModel'").ID.unique(): - NEW_ID = str(uuid4()) + NEW_ID = str(uuid.uuid4()) updated_ssh_id_map[OLD_ID] = NEW_ID logger.info(f"Assigned new UUID for updated SSH: {OLD_ID} -> {NEW_ID}") @@ -273,200 +263,63 @@ def create_sv_and_updated_ssh(merged_model, original_models, models_as_triplets, sv_data = sv_data.replace(updated_ssh_id_map) # Add SSH supersedes reference to old SSH - ssh_supersedes_data = pandas.DataFrame([{"ID": item[1], "KEY": "Model.Supersedes", "VALUE": item[0]} for item in updated_ssh_id_map.items()]) + ssh_supersedes_data = pd.DataFrame([{"ID": item[1], "KEY": "Model.Supersedes", "VALUE": item[0]} for item in updated_ssh_id_map.items()]) ssh_supersedes_data['INSTANCE_ID'] = ssh_data.query("KEY == 'Type'").merge(ssh_supersedes_data.ID)['INSTANCE_ID'] ssh_data = ssh_data.update_triplet_from_triplet(ssh_supersedes_data) # Update SSH metadata - ssh_data = update_FullModel_from_OpdmObject(ssh_data, opdm_object_meta) + ssh_data = update_header_from_opdm_object(ssh_data, opdm_object_meta) # Update SSH filenames filename_mask = "{scenarioTime:%Y%m%dT%H%MZ}_{processType}_{mergingEntity}-{domain}-{forEntity}_{messageType}_{version:03d}" ssh_data = triplets.cgmes_tools.update_filename_from_FullModel(ssh_data, filename_mask=filename_mask) - return sv_data, ssh_data - - -def fix_sv_shunts(sv_data, models_as_triplets): - """Remove Shunt Sections for EQV Shunts""" - - equiv_shunt = models_as_triplets.query("KEY == 'Type' and VALUE == 'EquivalentShunt'") - if len(equiv_shunt) > 0: - shunts_to_remove = sv_data.merge( - sv_data.query("KEY == 'SvShuntCompensatorSections.ShuntCompensator'").merge(equiv_shunt.ID, left_on='VALUE', - right_on="ID", how='inner', - suffixes=('', '_EQVShunt')).ID) - if len(shunts_to_remove) > 0: - logger.warning(f'Removing invalid SvShuntCompensatorSections for EquivalentShunt') - sv_data = triplets.rdf_parser.remove_triplet_from_triplet(sv_data, shunts_to_remove) + return sv_data, ssh_data, opdm_object_meta - return sv_data - - -def fix_sv_tapsteps(sv_data, ssh_data): - """Fix SV - Remove Shunt Sections for EQV Shunts""" - - ssh_tap_steps = ssh_data.query("KEY == 'TapChanger.step'") - sv_tap_steps = sv_data.query("KEY == 'SvTapStep.TapChanger'") - missing_sv_tap_steps = ssh_tap_steps.merge(sv_tap_steps[['VALUE']], left_on='ID', right_on="VALUE", how='left', indicator=True, suffixes=('', '_SV')).query("_merge == 'left_only'") +def ensure_paired_equivalent_injection_compatibility(network: pypowsybl.network): + """Where there are paired boundary points, equivalent injections need to be modified to comply + LEVEL7 rule PairedEICompatibility - tap_steps_to_be_added = [] - SV_INSTANCE_ID = sv_data.INSTANCE_ID.iloc[0] - for tap_changer in missing_sv_tap_steps.itertuples(): - ID = str(uuid4()) - logger.warning( - f'Missing SvTapStep for {tap_changer.ID}, adding SvTapStep {ID} and taking tap value {tap_changer.VALUE} from SSH') - tap_steps_to_be_added.extend([ - (ID, 'Type', 'SvTapStep', SV_INSTANCE_ID), - (ID, 'SvTapStep.TapChanger', tap_changer.ID, SV_INSTANCE_ID), - (ID, 'SvTapStep.position', tap_changer.VALUE, SV_INSTANCE_ID), - ]) - - sv_data = pandas.concat([sv_data, pandas.DataFrame(tap_steps_to_be_added, columns=['ID', 'KEY', 'VALUE', 'INSTANCE_ID'])], ignore_index=True) - - return sv_data - - -def configure_paired_boundarypoint_injections(data): - """Where there are paired boundary points, eqivalent injections need to be modified - Set P and Q to 0 - so that no additional consumption or prduction is on tieline - Set voltage control off - so that no additional consumption or prduction is on tieline - Set terminal to connected - to be sure we have paired connected injections at boundary point + Set P and Q to 0 - so that no additional consumption or production is on tie line """ + logger.info("Configuring paired boundary points equivalent injections: p0/q0 = 0.0") + dangling_lines = network.get_dangling_lines(all_attributes=True) + paired_dangling_lines = dangling_lines[dangling_lines['paired'] == True] + if paired_dangling_lines.empty: + logger.warning(f"No paired dangling lines found in network model") + return network - boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] - #boundary_points = data.type_tableview("ConnectivityNode").reset_index().query("`ConnectivityNode.boundaryPoint` == 'true'") - boundary_points = boundary_points.merge(data.type_tableview("Terminal").reset_index(), left_on="ID", right_on="Terminal.ConnectivityNode", suffixes=('_ConnectivityNode', '_Terminal')) + # Set p0/q0 to 0 for all paired dangling lines + _updated_p0 = pd.Series(0, index=paired_dangling_lines.index) + _updated_q0 = pd.Series(0, index=paired_dangling_lines.index) + network.update_dangling_lines(id=paired_dangling_lines.index, p0=_updated_p0, q0=_updated_q0) - injections = data.type_tableview('EquivalentInjection').reset_index().merge(boundary_points, left_on="ID", right_on='Terminal.ConductingEquipment', suffixes=('_ConnectivityNode', '')) + return network - # Get paired injections at boundary points - paired_injections = injections.groupby("Terminal.ConnectivityNode").filter(lambda x: len(x) == 2) - # Set terminal status - updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) - updated_terminal_status["KEY"] = "ACDCTerminal.connected" - updated_terminal_status["VALUE"] = "true" +def ensure_paired_boundary_line_connectivity(network: pypowsybl.network): + logger.info("Aligning paired boundary lines connection status") + dangling_lines = network.get_dangling_lines(all_attributes=True) + paired_dangling_lines = dangling_lines[dangling_lines['paired'] == True] + if paired_dangling_lines.empty: + logger.warning(f"No paired dangling lines found in network model") + return network - # Set Regulation off - updated_regulation_status = paired_injections[["ID"]].copy() - updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" - updated_regulation_status["VALUE"] = "false" + # Identify dangling line pairs where the 'connected' status is inconsistent within each pairing_key group + mask = paired_dangling_lines.groupby('pairing_key')['connected'].transform(lambda s: s.nunique() > 1) + mismatched_dangling_lines = paired_dangling_lines[mask] + logger.info(f"Boundary lines with non-matching connection status: {mismatched_dangling_lines['pairing_key'].unique().tolist()}") - # Set P to 0 - updated_p_value = paired_injections[["ID"]].copy() - updated_p_value["KEY"] = "EquivalentInjection.p" - updated_p_value["VALUE"] = 0 + # Set all mismatched lines to disconnected (False) + _connected = pd.Series(data=False, index=mismatched_dangling_lines.index) + network.update_dangling_lines(id=mismatched_dangling_lines.index, connected=_connected) - # Set Q to 0 - updated_q_value = paired_injections[["ID"]].copy() - updated_q_value["KEY"] = "EquivalentInjection.q" - updated_q_value["VALUE"] = 0 + # Log each change + for i, row in mismatched_dangling_lines.iterrows(): + logger.info(f"Changed status of dangling line {row['name']}: {row['connected']} -> False") - return data.update_triplet_from_triplet(pandas.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) - - -def configure_paired_boundarypoint_injections_by_nodes(data): - """Where there are paired boundary points, eqivalent injections need to be modified - Set P and Q to 0 - so that no additional consumption or prduction is on tieline - Set voltage control off - so that no additional consumption or prduction is on tieline - Set terminal to connected - to be sure we have paired connected injections at boundary point - NOTE THAT THIS IS COPY FROM 'configure_paired_boundarypoint_injections' - In some models terminals are missing references to ConnectivityNodes - """ - connectivity_boundary_points = data.query("KEY == 'ConnectivityNode.boundaryPoint' and VALUE == 'true'")[["ID"]] - topological_boundary_points = data.query("KEY == 'TopologicalNode.boundaryPoint' and VALUE == 'true'")[["ID"]] - try: - terminals = data.type_tableview("Terminal").reset_index()[['ID', - 'Terminal.ConductingEquipment', - 'Terminal.ConnectivityNode', - 'Terminal.TopologicalNode']] - except KeyError: - terminals = data.type_tableview("Terminal").reset_index()[['ID', - 'Terminal.ConductingEquipment', - 'Terminal.TopologicalNode']] - injections = data.type_tableview('EquivalentInjection').reset_index()[['ID', - # 'EquivalentInjection.p', - # 'EquivalentInjection.q', - # 'EquivalentInjection.regulationStatus' - ]] - topological_boundary_points = topological_boundary_points.merge(terminals, - left_on="ID", - right_on="Terminal.TopologicalNode", - suffixes=('_TopologicalNode', '_Terminal')) - topological_injections = injections.merge(topological_boundary_points, - left_on="ID", - right_on='Terminal.ConductingEquipment', - suffixes=('_ConnectivityNode', '')) - paired_topological_injections = (topological_injections.groupby("Terminal.TopologicalNode") - .filter(lambda x: len(x) == 2)) - paired_injections = paired_topological_injections - if 'Terminal.ConnectivityNode' in terminals: - connectivity_boundary_points = connectivity_boundary_points.merge(terminals, - left_on="ID", - right_on="Terminal.ConnectivityNode", - suffixes=('_ConnectivityNode', '_Terminal')) - connectivity_injections = injections.merge(connectivity_boundary_points, - left_on="ID", - right_on='Terminal.ConductingEquipment', - suffixes=('_TopologicalNode', '')) - - paired_connectivity_injections = (connectivity_injections.groupby("Terminal.ConnectivityNode") - .filter(lambda x: len(x) == 2)) - merged_injections = paired_connectivity_injections.merge(paired_topological_injections, - on='ID', - how='outer', - indicator=True, - suffixes=('_CN', '_TN')) - only_connectivity_injections = merged_injections[merged_injections['_merge'] == 'left_only'] - only_topological_injections = merged_injections[merged_injections['_merge'] == 'right_only'] - if len(only_connectivity_injections.index) != 0 or len(only_topological_injections.index) == 0: - paired_injections = paired_connectivity_injections - else: - logger.warning(f"Mismatch of finding paired injections from topological nodes and connectivity nodes") - else: - logger.warning(f"Terminals do not contain Connectivity nodes") - # Set terminal status - updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) - updated_terminal_status["KEY"] = "ACDCTerminal.connected" - updated_terminal_status["VALUE"] = "true" - - # Set Regulation off - updated_regulation_status = paired_injections[["ID"]].copy() - updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" - updated_regulation_status["VALUE"] = "false" - - # Set P to 0 - updated_p_value = paired_injections[["ID"]].copy() - updated_p_value["KEY"] = "EquivalentInjection.p" - updated_p_value["VALUE"] = 0 - - # Set Q to 0 - updated_q_value = paired_injections[["ID"]].copy() - updated_q_value["KEY"] = "EquivalentInjection.q" - updated_q_value["VALUE"] = 0 - - return data.update_triplet_from_triplet(pandas.concat([updated_terminal_status, updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) - - -def export_to_cgmes_zip(triplets: list): - namespace_map = { - "rdf": "http://www.w3.org/1999/02/22-rdf-syntax-ns#", - "cim": "http://iec.ch/TC57/2013/CIM-schema-cim16#", - "md": "http://iec.ch/TC57/61970-552/ModelDescription/1#", - "entsoe": "http://entsoe.eu/CIM/SchemaExtension/3/1#", - } - - # with open('../../config/cgm_worker/CGMES_v2_4_15_2014_08_07.json', 'r') as file_object: - rdf_map = json.load(config.paths.cgm_worker.CGMES_v2_4_15_2014_08_07) - - return pandas.concat(triplets, ignore_index=True).export_to_cimxml(rdf_map=rdf_map, - namespace_map=namespace_map, - export_undefined=False, - export_type="xml_per_instance_zip_per_xml", - debug=False, - export_to_memory=True) + return network def generate_merge_report(merged_model: object, task: dict): @@ -603,194 +456,121 @@ def filter_models(models: list, included_models: list | str = None, excluded_mod return filtered_models -def get_boundary_nodes_between_igms(model_data: list | pandas.DataFrame): - """ - Filters out nodes that are between the igms (mentioned at least 2 igms) - :param model_data: input models - : return series of node ids - """ - model_data = get_opdm_data_from_models(model_data=model_data) - all_boundary_nodes = model_data[(model_data['KEY'] == 'TopologicalNode.boundaryPoint') & - (model_data['VALUE'] == 'true')] - # Get boundary nodes that exist in igms - merged = pandas.merge(all_boundary_nodes, - model_data[(model_data['KEY'] == 'SvVoltage.TopologicalNode')], - left_on='ID', right_on='VALUE', suffixes=('_y', '')) - # Get duplicates (all of them) then duplicated values. keep=False marks all duplicates True, 'first' marks first - # occurrence to false, 'last' marks last occurrence to false. If any of them is used then in case duplicates are 2 - # then 1 is retrieved, if duplicates >3 then duplicates-1 retrieved. So, get all the duplicates and as a second - # step, drop the duplicates - merged = (merged[merged.duplicated(['VALUE'], keep=False)]).drop_duplicates(subset=['VALUE']) - in_several_igms = (merged["VALUE"]).to_frame().rename(columns={'VALUE': 'ID'}) - return in_several_igms - - -def take_best_match_for_sv_voltage(input_data, column_name: str = 'v', to_keep: bool = True): - """ - Returns one row for with sv voltage id for topological node - 1) Take the first - 2) If first is zero take first non-zero row if exists - :param input_data: input dataframe - :param column_name: name of the column - :param to_keep: either to keep or discard a value - """ - first_row = input_data.iloc[0] - if to_keep: - remaining_rows = input_data[input_data[column_name] != 0] - if first_row[column_name] == 0 and not remaining_rows.empty: - first_row = remaining_rows.iloc[0] +def update_model_outages(merged_model: object, tso_list: list, scenario_datetime: str, time_horizon: str): + + area_map = {"LITGRID": "Lithuania", "AST": "Latvia", "ELERING": "Estonia"} + outage_areas = [area_map.get(item, item) for item in tso_list] + + elk_service = elastic.Elastic() + + # Get outage eic-mrid mapping + mrid_map = elk_service.get_docs_by_query(index='config-network*', query={"match_all": {}}, size=10000) + mrid_map['mrid'] = mrid_map['mrid'].str.lstrip('_') + + # Get latest UAP parse date + if time_horizon == 'MO': + merge_type = "Month" + elif time_horizon == 'YR': + merge_type = "Year" else: - remaining_rows = input_data[input_data[column_name] == 0] - if first_row[column_name] != 0 and not remaining_rows.empty: - first_row = remaining_rows.iloc[0] - return first_row + merge_type = "Week" + + body = {"size": 1, "query": {"bool": {"must": [{"match": {"Merge": merge_type}}]}}, + "sort": [{"reportParsedDate": {"order": "desc"}}], "fields": ["reportParsedDate"]} + last_uap_version = elk_service.client.search(index='opc-outages-baltics*', body=body)['hits']['hits'][0]['fields']['reportParsedDate'][0] + + # Query for latest outage UAP + uap_query = {"bool": {"must": [{"match": {"reportParsedDate": f"{last_uap_version}"}}, + {"match": {"Merge": merge_type}}]}} + uap_outages = elk_service.get_docs_by_query(index='opc-outages-baltics*', query=uap_query, size=10000) + uap_outages = uap_outages.merge(mrid_map[['eic', 'mrid']], how='left', on='eic', indicator=True).rename(columns={"mrid": 'grid_id'}) + unmapped_outages = uap_outages[uap_outages['_merge'] == 'left_only'] + + if not unmapped_outages.empty: + logger.warning(f"Unable to map following outage mRIDs: {unmapped_outages['name'].values}") + + # Filter outages according to model scenario date and replaced area + filtered_outages = uap_outages[(uap_outages['start_date'] <= scenario_datetime) & (uap_outages['end_date'] >= scenario_datetime)] + filtered_outages = filtered_outages[filtered_outages['Area'].isin(outage_areas)] + mapped_outages = filtered_outages[~filtered_outages['grid_id'].isna()] + + # Get disconnected elements in network model + model_outages = pd.DataFrame(get_model_outages(network=merged_model.network)) + mapped_model_outages = pd.merge(model_outages, mrid_map, left_on='grid_id', right_on='mrid', how='inner') + model_area_map = {"LITGRID": "LT", "AST": "LV", "ELERING": "EE"} + model_outage_areas = [model_area_map.get(item, item) for item in tso_list] + filtered_model_outages = mapped_model_outages[mapped_model_outages['country'].isin(model_outage_areas)] + + # Include cross-border lines for reconnection (both dangling lines) + dangling_lines = get_network_elements(network=merged_model.network, + element_type=pypowsybl.network.ElementType.DANGLING_LINE).reset_index(names=['grid_id']) + border_lines = dangling_lines[dangling_lines['pairing_key'].isin(model_outages['pairing_key'])] + relevant_border_lines = border_lines[border_lines['country'].isin(['LT', 'LV', 'EE'])] + additional_dangling_lines = dangling_lines[dangling_lines['pairing_key'].isin(relevant_border_lines['pairing_key'])] + + # Merged dataframe of network elements to be reconnected + filtered_model_outages = pd.concat([filtered_model_outages, additional_dangling_lines]).drop_duplicates(subset='grid_id') + filtered_model_outages = filtered_model_outages.where(pd.notnull(filtered_model_outages), None) + + # rename columns + filtered_model_outages = filtered_model_outages.copy()[['name', 'grid_id', 'eic']].rename(columns={'grid_id': 'mrid'}) + mapped_outages = mapped_outages.copy()[['name', 'grid_id', 'eic']].rename(columns={'grid_id': 'mrid'}) + + logger.info("Updating outages on merged model") + + # Reconnecting outages from network-config list + outages_updated = {} + for index, outage in filtered_model_outages.iterrows(): + try: + if merged_model.network.connect(outage['mrid']): + logger.info(f"Successfully reconnected: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages = True + outage_dict = outage.to_dict() + outage_dict.update({'status': 'connected'}) + outages_updated[outage_dict['mrid']] = outage_dict + else: + if uap_outages['grid_id'].str.contains(outage['mrid']).any(): + logger.info(f"Element is already connected: {outage['name']} [mrid: {outage['mrid']}]") + else: + logger.error(f"Failed to connect element: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + except Exception as e: + logger.error((e, outage['name'])) + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + merged_model.outages = False + continue + + # Applying outages from UAP + for index, outage in mapped_outages.iterrows(): + try: + if merged_model.network.disconnect(outage['mrid']): + logger.info(f"Successfully disconnected: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages = True + outage_dict = outage.to_dict() + outage_dict.update({'status': 'disconnected'}) + outages_updated[outage_dict['mrid']] = outage_dict + else: + if uap_outages['grid_id'].str.contains(outage['mrid']).any(): + logger.info(f"Element is already in outage: {outage['name']} [mrid: {outage['mrid']}]") + else: + logger.error(f"Failed to disconnect element: {outage['name']} [mrid: {outage['mrid']}]") + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + except Exception as e: + logger.error((e, outage['name'])) + merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) + merged_model.outages = False + continue + + # Keep only important keys of updated outages + merged_model.outages_updated = list(outages_updated.values()) + + if merged_model.outages_unmapped: + merged_model.outages = False + return merged_model -def remove_duplicate_sv_voltages(cgm_sv_data, original_data): - """ - Pypowsybl 1.6.0 provides multiple sets of SvVoltage values for the topological nodes that are boundary nodes (from - each IGM side that uses the corresponding boundary node). So this is a hack that removes one of them (preferably the - one that is zero). - :param cgm_sv_data: merged SV profile from where duplicate SvVoltage values are removed - :param original_data: will be used to get boundary node ids - :return updated merged SV profile - """ - # Check that models are in triplets - some_data = get_opdm_data_from_models(model_data=original_data) - # Get ids of boundary nodes that are shared by several igms - in_several_igms = (get_boundary_nodes_between_igms(model_data=some_data)) - # Get SvVoltage Ids corresponding to shared boundary nodes - sv_voltage_ids = pandas.merge(cgm_sv_data[cgm_sv_data['KEY'] == 'SvVoltage.TopologicalNode'], - in_several_igms.rename(columns={'ID': 'VALUE'}), on='VALUE') - # Get SvVoltage voltage values for corresponding SvVoltage Ids - sv_voltage_values = pandas.merge(cgm_sv_data[cgm_sv_data['KEY'] == 'SvVoltage.v'][['ID', 'VALUE']]. - rename(columns={'VALUE': 'SvVoltage.v'}), - sv_voltage_ids[['ID', 'VALUE']]. - rename(columns={'VALUE': 'SvVoltage.SvTopologicalNode'}), on='ID') - # Just in case convert the values to numeric - sv_voltage_values[['SvVoltage.v']] = (sv_voltage_values[['SvVoltage.v']].apply(lambda x: x.apply(Decimal))) - # Group by topological node id and by some logic take SvVoltage that will be dropped - voltages_to_keep = (sv_voltage_values.groupby(['SvVoltage.SvTopologicalNode']). - apply(lambda x: take_best_match_for_sv_voltage(input_data=x, - column_name='SvVoltage.v', - to_keep=True), include_groups=False)) - voltages_to_discard = sv_voltage_values.merge(voltages_to_keep['ID'], on='ID', how='left', indicator=True) - voltages_to_discard = voltages_to_discard[voltages_to_discard['_merge'] == 'left_only'] - if not voltages_to_discard.empty: - logger.info(f"Removing {len(voltages_to_discard.index)} duplicate voltage levels from boundary nodes") - sv_voltages_to_remove = pandas.merge(cgm_sv_data, voltages_to_discard['ID'].to_frame(), on='ID') - cgm_sv_data = triplets.rdf_parser.remove_triplet_from_triplet(cgm_sv_data, sv_voltages_to_remove) - return cgm_sv_data - - -def check_and_fix_dependencies(cgm_sv_data, cgm_ssh_data, original_data): - """ - Seems that pypowsybl ver 1.6.0 managed to get rid of dependencies in exported file. This gathers them from - SSH profiles and from the original models - :param cgm_sv_data: merged SV profile that is missing the dependencies - :param cgm_ssh_data: merged SSH profiles, will be used to get SSH dependencies - :param original_data: original models, will be used to get TP dependencies - :return updated merged SV profile - """ - some_data = get_opdm_data_from_models(model_data=original_data) - tp_file_ids = some_data[(some_data['KEY'] == 'Model.profile') & (some_data['VALUE'].str.contains('Topology'))] - - ssh_file_ids = cgm_ssh_data[(cgm_ssh_data['KEY'] == 'Model.profile') & - (cgm_ssh_data['VALUE'].str.contains('SteadyStateHypothesis'))] - dependencies = pandas.concat([tp_file_ids, ssh_file_ids], ignore_index=True, sort=False) - existing_dependencies = cgm_sv_data[cgm_sv_data['KEY'] == 'Model.DependentOn'] - dependency_difference = existing_dependencies.merge(dependencies[['ID']].rename(columns={'ID': 'VALUE'}), - on='VALUE', how='outer', indicator=True) - if not dependency_difference.query('_merge == "right_only"').empty: - cgm_sv_data = triplets.rdf_parser.remove_triplet_from_triplet(cgm_sv_data, existing_dependencies) - full_model_id = cgm_sv_data[(cgm_sv_data['KEY'] == 'Type') & (cgm_sv_data['VALUE'] == 'FullModel')] - dependencies_to_update = dependency_difference.query('_merge != "left_only"') - logger.info(f"Mismatch of dependencies. Inserting {len(dependencies_to_update.index)} " - f"dependencies to SV profile") - new_dependencies = dependencies_to_update[['VALUE']].copy().reset_index(drop=True) - new_dependencies.loc[:, 'KEY'] = 'Model.DependentOn' - new_dependencies.loc[:, 'ID'] = full_model_id['ID'].iloc[0] - new_dependencies.loc[:, 'INSTANCE_ID'] = full_model_id['INSTANCE_ID'].iloc[0] - cgm_sv_data = triplets.rdf_parser.update_triplet_from_triplet(cgm_sv_data, new_dependencies) - return cgm_sv_data - - -def remove_small_islands(solved_data, island_size_limit): - small_island = pandas.DataFrame(solved_data.query("KEY == 'TopologicalIsland.TopologicalNodes'").ID.value_counts()).reset_index().query("count <= @island_size_limit") - solved_data = triplets.rdf_parser.remove_triplet_from_triplet(solved_data, small_island, columns=["ID"]) - logger.info(f"Removed {len(small_island)} island(s) with size <= {island_size_limit}") - return solved_data - - -def disconnect_equipment_if_flow_sum_not_zero(cgm_sv_data, - cgm_ssh_data, - original_data, - equipment_name: str = "ConformLoad", - sv_injection_limit: float = SV_INJECTION_LIMIT): - """ - If there is a mismatch of flows at topological nodes it tries to switch of and set flows at terminals - indicated by equipment_name to original values. - The idea is that when loadflow calculation fails at some island, the results are still being updated and as - currently there is not a better way to find the islands-> nodes -> terminals on which it fails then the HACK - is to try to set them back to original values. - NOTE THAT IT NOT ONLY SETS THE VALUES TO OLD ONES BUT ALSO DISCONNECTS IT FROM TERMINAL - :param cgm_ssh_data: merged SSH profile (needed to switch the terminals of) - :param cgm_sv_data: merged SV profile (needed to set the flows for terminals) - :param original_data: IGMs (triplets, dictionary) - :param equipment_name: name of the equipment. CURRENTLY, IT IS USED FOR CONFORM LOAD - :param sv_injection_limit: threshold for deciding whether the node is violated by sum of flows - :return updated merged SV and SSH profiles - """ - original_data = get_opdm_data_from_models(model_data=original_data) - # Get power flow after lf - power_flow = cgm_sv_data.type_tableview('SvPowerFlow')[['SvPowerFlow.Terminal', 'SvPowerFlow.p', 'SvPowerFlow.q']] - # Get terminals - terminals = original_data.type_tableview('Terminal').rename_axis('Terminal').reset_index() - terminals = terminals[['Terminal', 'Terminal.ConductingEquipment', 'Terminal.TopologicalNode']] - # Calculate summed flows per topological node - flows_summed = ((power_flow.merge(terminals, left_on='SvPowerFlow.Terminal', right_on='Terminal', how='left') - .groupby('Terminal.TopologicalNode')[['SvPowerFlow.p', 'SvPowerFlow.q']] - .sum()).rename_axis('Terminal.TopologicalNode').reset_index()) - # Get topological nodes that have mismatch - nok_nodes = flows_summed[(abs(flows_summed['SvPowerFlow.p']) > sv_injection_limit) | - (abs(flows_summed['SvPowerFlow.q']) > sv_injection_limit)][['Terminal.TopologicalNode']] - # Merge terminals with summed flows at nodes - terminals_nodes = terminals.merge(flows_summed, on='Terminal.TopologicalNode', how='left') - # Get equipment names - equipment_names = (original_data.query('KEY == "Type"')[['ID', 'VALUE']] - .drop_duplicates().rename(columns={'ID': 'Terminal.ConductingEquipment', - 'VALUE': 'Equipment_name'})) - # Merge terminals with equipment names - terminals_equipment = terminals_nodes.merge(equipment_names, on='Terminal.ConductingEquipment', how='left') - # Get equipment lines corresponding to nodes that had mismatch - if not nok_nodes.empty: - logger.error(f"For {len(nok_nodes.index)} topological nodes, the sum of flows is over {sv_injection_limit}") - nok_lines = (terminals_equipment.merge(nok_nodes, on='Terminal.TopologicalNode') - .sort_values(by=['Terminal.TopologicalNode'])) - nok_loads = nok_lines[nok_lines['Equipment_name'] == equipment_name] - if not nok_loads.empty: - logger.warning(f"Switching off {len(nok_loads.index)} terminals as they contain {equipment_name}") - # Copy values from original models - old_power_flows = original_data.type_tableview('SvPowerFlow')[['SvPowerFlow.Terminal', - 'SvPowerFlow.p', 'SvPowerFlow.q']] - old_power_flows = (old_power_flows - .merge(nok_loads[['Terminal']].rename(columns={'Terminal': 'SvPowerFlow.Terminal'}), - on='SvPowerFlow.Terminal')) - new_power_flows = cgm_sv_data.type_tableview('SvPowerFlow')[['SvPowerFlow.Terminal', 'Type']] - new_power_flows = (new_power_flows.reset_index().merge(old_power_flows, on='SvPowerFlow.Terminal') - .set_index('ID')) - # Update values in SV profile - cgm_sv_data = triplets.rdf_parser.update_triplet_from_tableview(cgm_sv_data, new_power_flows) - # Just in case disconnect those things also - # The following part can cause 'Disconnected Terminal' error - # terminals_in_ssh = cgm_ssh_data[cgm_ssh_data["KEY"].str.contains("Terminal.connected")].merge( - # nok_loads[["Terminal"]].rename(columns={'Terminal': 'ID'}), on='ID') - # terminals_in_ssh.loc[:, 'VALUE'] = 'false' - # cgm_ssh_data = triplets.rdf_parser.update_triplet_from_triplet(cgm_ssh_data, terminals_in_ssh) - return cgm_sv_data, cgm_ssh_data - - -def calculate_intraday_time_horizon(scenario_datetime, task_creation_time): + +def set_intraday_time_horizon(scenario_datetime, task_creation_time): """ Finds time difference between task creation time and scenario timestamp. Converts it to hours and finds the hour number corresponding to intraday run (number of hours that scenario timestamp is ahead from task creation time) @@ -808,8 +588,8 @@ def calculate_intraday_time_horizon(scenario_datetime, task_creation_time): """ max_time_horizon_value = 36 calculated_time_horizon = '01' # DEFAULT VALUE, CHANGE THIS - _task_creation_time = helper_parse_datetime(task_creation_time, keep_timezone=False) - _scenario_datetime = helper_parse_datetime(scenario_datetime, keep_timezone=False) + _task_creation_time = parse_datetime(task_creation_time, keep_timezone=False) + _scenario_datetime = parse_datetime(scenario_datetime, keep_timezone=False) time_diff = _scenario_datetime - _task_creation_time if 0 <= time_diff.days <= 1: time_horizon_actual = math.floor((time_diff.days * 24 * 3600 + time_diff.seconds) / 3600) @@ -820,9 +600,268 @@ def calculate_intraday_time_horizon(scenario_datetime, task_creation_time): return calculated_time_horizon +def check_net_interchanges(cgm_sv_data, cgm_ssh_data, original_models, fix_errors: bool = False, threshold: float = None): + """ + An attempt to calculate the net interchange 2 values and check them against those provided in ssh profiles + :param cgm_sv_data: merged sv profile + :param cgm_ssh_data: merged ssh profile + :param original_models: igms in triplets + :param fix_errors: injects new calculated flows into merged ssh profiles + :param threshold: specify threshold if needed + :return (updated) ssh profiles + """ + try: + control_areas = (original_models.type_tableview('ControlArea') + .rename_axis('ControlArea') + .reset_index())[['ControlArea', 'ControlArea.netInterchange', 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', 'IdentifiedObject.name']] + except KeyError: + control_areas = original_models.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + ssh_areas = cgm_ssh_data.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + control_areas = control_areas.merge(ssh_areas, on='ControlArea')[['ControlArea', 'ControlArea.netInterchange', + 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', + 'IdentifiedObject.name']] + tie_flows = (original_models.type_tableview('TieFlow') + .rename_axis('TieFlow').rename(columns={'TieFlow.ControlArea': 'ControlArea', + 'TieFlow.Terminal': 'Terminal'}) + .reset_index())[['ControlArea', 'Terminal', 'TieFlow.positiveFlowIn']] + tie_flows = tie_flows.merge(control_areas[['ControlArea']], on='ControlArea') + try: + terminals = (original_models.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal', 'ACDCTerminal.connected']] + except KeyError: + terminals = (original_models.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal']] + tie_flows = tie_flows.merge(terminals, on='Terminal') + try: + power_flows_pre = (original_models.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + tie_flows = tie_flows.merge(power_flows_pre, on='Terminal', how='left') + except Exception as error: + logger.error(f"Was not able to get tie flows from original models with exception: {error}") + power_flows_post = (cgm_sv_data.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + + tie_flows = tie_flows.merge(power_flows_post, on='Terminal', how='left', + suffixes=('_pre', '_post')) + try: + tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p_pre', 'SvPowerFlow.p_post']] + .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) + .rename_axis('ControlArea').reset_index()) + except KeyError: + tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p']] + .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) + .rename_axis('ControlArea').reset_index()) + tie_flows_grouped = tie_flows_grouped.rename(columns={'SvPowerFlow.p': 'SvPowerFlow.p_post'}) + tie_flows_grouped = control_areas.merge(tie_flows_grouped, on='ControlArea') + if threshold and threshold > 0: + tie_flows_grouped['Exceeded'] = (abs(tie_flows_grouped['ControlArea.netInterchange'] + - tie_flows_grouped['SvPowerFlow.p_post']) > threshold) + else: + tie_flows_grouped['Exceeded'] = (abs(tie_flows_grouped['ControlArea.netInterchange'] + - tie_flows_grouped['SvPowerFlow.p_post']) > + tie_flows_grouped['ControlArea.pTolerance']) + net_interchange_errors = tie_flows_grouped[tie_flows_grouped.eval('Exceeded')] + if not net_interchange_errors.empty: + logger.warning(f"Found {len(net_interchange_errors.index)} possible net interchange_2 problems over {threshold}") + # Apply modification + if fix_errors: + logger.warning(f"Updating {len(net_interchange_errors.index)} interchanges to new values") + new_areas = cgm_ssh_data.type_tableview('ControlArea').reset_index()[['ID', + 'ControlArea.pTolerance', 'Type']] + new_areas = new_areas.merge(net_interchange_errors[['ControlArea', 'SvPowerFlow.p_post']] + .rename(columns={'ControlArea': 'ID', + 'SvPowerFlow.p_post': 'ControlArea.netInterchange'}), on='ID') + cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(cgm_ssh_data, new_areas) + + return cgm_ssh_data + + +def check_non_boundary_equivalent_injections(cgm_sv_data, + cgm_ssh_data, + original_models, + threshold: float = 0, + fix_errors: bool = False): + """ + Checks equivalent injections that are not on boundary topological nodes + :param cgm_sv_data: merged SV profile + :param cgm_ssh_data: merged SSH profile + :param original_models: igms in triplets + :param threshold: threshold for checking + :param fix_errors: if true then copies values from sv profile to ssh profile + :return cgm_ssh_data + """ + boundary_nodes = original_models.query('KEY == "TopologicalNode.boundaryPoint" & VALUE == "true"')[['ID']] + terminals = (original_models.type_tableview('Terminal').rename_axis('SvPowerFlow.Terminal').reset_index() + .merge(boundary_nodes.rename(columns={'ID': 'Terminal.TopologicalNode'}), + on='Terminal.TopologicalNode', how='outer', indicator=True))[['SvPowerFlow.Terminal', + 'Terminal.ConductingEquipment', + '_merge']] + terminals = terminals[terminals['_merge'] == 'left_only'][['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] + return check_all_kind_of_injections(cgm_sv_data=cgm_sv_data, + cgm_ssh_data=cgm_ssh_data, + original_models=original_models, + injection_name='EquivalentInjection', + fields_to_check={'SvPowerFlow.p': 'EquivalentInjection.p'}, + threshold=threshold, + terminals=terminals, + fix_errors=fix_errors) + + +def check_all_kind_of_injections(cgm_sv_data, + cgm_ssh_data, + original_models, + injection_name: str = 'ExternalNetworkInjection', + fields_to_check: dict = None, + fix_errors: bool = False, + threshold: float = 0, + terminals: pd.DataFrame = None, + report_sum: bool = True): + """ + Compares the given cgm ssh injection values to the corresponding sv powerflow values in cgm sv + :param cgm_sv_data: merged SV profile + :param cgm_ssh_data: merged SSH profile + :param original_models: igms in triplets + :param injection_name: name of the injection + :param fields_to_check: dictionary where key is the field in powerflow and value is the field in injection + :param fix_errors: if true then copies values from sv profile to ssh profile + :param threshold: max allowed mismatch + :param terminals: optional, can give dataframe of terminals as input + :param report_sum: if true prints sum of injections and powerflows to console + :return cgm_ssh_data + """ + if not fields_to_check: + return cgm_ssh_data + + fixed_fields = ['ID'] + try: + original_injections = original_models.type_tableview(injection_name).reset_index() + injections = cgm_ssh_data.type_tableview(injection_name).reset_index() + except AttributeError: + logger.info(f"SSH profile doesn't contain data about {injection_name}") + return cgm_ssh_data + try: + injections_reduced = injections[[*fixed_fields, *fields_to_check.values()]] + original_injections_reduced = original_injections[[*fixed_fields, *fields_to_check.values()]] + except KeyError as ke: + logger.info(f"{injection_name} tableview got error: {ke}") + return cgm_ssh_data + injections_reduced = injections_reduced.merge(original_injections_reduced, on='ID', suffixes=('', '_org')) + if terminals is None: + terminals = (original_models.type_tableview('Terminal') + .rename_axis('SvPowerFlow.Terminal') + .reset_index())[['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] + flows = (cgm_sv_data.type_tableview('SvPowerFlow') + .reset_index())[[*['SvPowerFlow.Terminal'], *fields_to_check.keys()]] + terminals = terminals.merge(flows, on='SvPowerFlow.Terminal') + terminals = terminals.merge(injections_reduced, left_on='Terminal.ConductingEquipment', right_on='ID') + + filtered_list = [] + for flow_field, injection_field in fields_to_check.items(): + filtered_list.append(terminals[abs(terminals[injection_field] - terminals[flow_field]) > threshold]) + if report_sum: + logger.info(f"IGM {injection_field} = {terminals[injection_field + '_org'].sum()} vs " + f"CGM {injection_field} = {terminals[injection_field].sum()} vs " + f"CGM {flow_field} = {terminals[flow_field].sum()}") + if not filtered_list: + return cgm_ssh_data + + filtered = pd.concat(filtered_list).drop_duplicates().reset_index(drop=True) + if not filtered.empty: + logger.warning(f"Found {len(filtered.index)} mismatches between {injection_name} and flow values on terminals") + # Apply modification + if fix_errors: + logger.info(f"Updating {injection_name} values from terminal flow values") + injections_update = injections.merge(filtered[[*fixed_fields, *fields_to_check.keys()]]) + injections_update = injections_update.drop(columns=fields_to_check.values()) + injections_update = injections_update.rename(columns=fields_to_check) + cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(data=cgm_ssh_data, + tableview=injections_update, + update=True, + add=False) + return cgm_ssh_data + + +def run_post_merge_processing(input_models: list, + exported_model: bytes, + opdm_object_meta: dict, + enable_temp_fixes: bool, + task_properties: dict = None, + ): + + # Load original input models to triplets + input_models_triplets = load_opdm_objects_to_triplets(opdm_objects=input_models) + + # Apply corrections to SV profile + sv_data = update_merged_model_sv(sv_data=exported_model, opdm_object_meta=opdm_object_meta) + + # Create update SSH + sv_data, ssh_data, opdm_object_meta = create_updated_ssh(models_as_triplets=input_models_triplets, + input_models = input_models, + sv_data=sv_data, + opdm_object_meta=opdm_object_meta) + fix_net_interchange_errors = False + if task_properties is not None: + fix_net_interchange_errors = task_properties.get('fix_net_interchange2', fix_net_interchange_errors) + + # Run temporary modifications on exported model + # Temporary fixes are applied to SV and SSH profiles + if enable_temp_fixes: + # TODO need to revise constantly + sv_data = temporary.remove_equivalent_shunt_section(sv_data, input_models_triplets) + sv_data = temporary.add_missing_sv_tap_steps(sv_data, ssh_data) + sv_data = temporary.remove_small_islands(sv_data, int(SMALL_ISLAND_SIZE)) + sv_data = temporary.remove_duplicate_sv_voltages(cgm_sv_data=sv_data, original_data=input_models_triplets) + sv_data = temporary.check_and_fix_dependencies(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=input_models_triplets) + # TODO following SSH profile fix should be removed once pypowsybl SSH export will be used + ssh_data = temporary.set_paired_boundary_injections_to_zero(original_models=input_models_triplets, + cgm_ssh_data=ssh_data) + + # Run injections check and apply modification if defined in configuration + injection_threshold = float(INJECTION_THRESHOLD) + net_interchange_threshold = float(NET_INTERCHANGE_THRESHOLD) + fix_injection_errors = json.loads(str(FIX_INJECTION_ERRORS).lower()) + + ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, + cgm_sv_data=sv_data, + original_models=input_models_triplets, + injection_name='EnergySource', + threshold=injection_threshold, + fields_to_check={'SvPowerFlow.p': 'EnergySource.activePower'}, + fix_errors=fix_injection_errors) + ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, + cgm_sv_data=sv_data, + original_models=input_models_triplets, + injection_name='ExternalNetworkInjection', + fields_to_check={'SvPowerFlow.p': 'ExternalNetworkInjection.p'}, + threshold=injection_threshold, + fix_errors=fix_injection_errors) + ssh_data = check_non_boundary_equivalent_injections(cgm_sv_data=sv_data, + cgm_ssh_data=ssh_data, + original_models=input_models_triplets, + threshold=injection_threshold, + fix_errors=fix_injection_errors) + + try: + ssh_data = check_net_interchanges(cgm_sv_data=sv_data, + cgm_ssh_data=ssh_data, + original_models=input_models_triplets, + fix_errors=fix_net_interchange_errors, + threshold=net_interchange_threshold) + except KeyError: + logger.warning(f"No fields for net interchange correction") + + return sv_data, ssh_data, opdm_object_meta + + if __name__ == "__main__": from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download + from emf.common.helpers.loadflow import load_network_model + from emf.common.loadflow_tool import loadflow_settings logging.basicConfig( format='%(levelname)-10s %(asctime)s.%(msecs)03d %(name)-30s %(funcName)-35s %(lineno)-5d: %(message)s', @@ -841,20 +880,8 @@ def calculate_intraday_time_horizon(scenario_datetime, task_creation_time): valid_models = get_latest_models_and_download(time_horizon, scenario_date, valid=True) latest_boundary = get_latest_boundary() - merged_model = load_model(valid_models + [latest_boundary]) - - # TODO - run other LF if default fails - solved_model = run_lf(merged_model, loadflow_settings=loadflow_settings.CGM_DEFAULT) - - # TODO - get version dynamically form ELK - sv_data, ssh_data = create_sv_and_updated_ssh(solved_model, valid_models, time_horizon, version, merging_area, merging_entity, mas) - - # Fix SV - sv_data = fix_sv_shunts(sv_data, valid_models) - sv_data = fix_sv_tapsteps(sv_data, ssh_data) - - # Package to in memory zip files - serialized_data = export_to_cgmes_zip([ssh_data, sv_data]) + merged_model = load_network_model(valid_models + [latest_boundary]) + solved_model = pypowsybl.loadflow.run_ac(merged_model, loadflow_settings=loadflow_settings.CGM_DEFAULT) # Export to OPDM from emf.common.integrations.opdm import OPDM diff --git a/emf/model_merger/model_merger.py b/emf/model_merger/model_merger.py index d80e5574..0386bf0c 100644 --- a/emf/model_merger/model_merger.py +++ b/emf/model_merger/model_merger.py @@ -1,5 +1,4 @@ import logging -import pandas import pypowsybl import config import json @@ -8,8 +7,7 @@ import datetime from dataclasses import dataclass, field from typing import List - -from emf.common.time_helper import parse_datetime +from emf.common.helpers.time import parse_datetime from io import BytesIO from zipfile import ZipFile from emf.common.config_parser import parse_app_properties @@ -17,17 +15,19 @@ from emf.common.integrations.object_storage.models import get_latest_boundary, get_latest_models_and_download from emf.common.integrations.object_storage.schedules import query_acnp_schedules, query_hvdc_schedules from emf.common.loadflow_tool import loadflow_settings -from emf.common.loadflow_tool.helper import opdmprofile_to_bytes, attr_to_dict +from emf.common.helpers.utils import attr_to_dict, convert_dict_str_to_bool +from emf.common.helpers.cgmes import export_to_cgmes_zip +from emf.common.helpers.opdm_objects import get_opdm_component_data_bytes +from emf.common.helpers.loadflow import load_network_model +from emf.common.helpers.tasks import update_task_status from emf.model_merger import merge_functions from emf.model_merger import scaler -from emf.model_merger.replacement import run_replacement, get_available_tsos, run_replacement_local -from emf.task_generator.task_generator import update_task_status +from emf.model_merger.replacement import run_replacement, get_tsos_available_in_storage +from emf.model_merger.temporary import handle_igm_ssh_vs_cgm_ssh_error from emf.common.logging.custom_logger import get_elk_logging_handler -# TODO - move this async solution to some common module from concurrent.futures import ThreadPoolExecutor from lxml import etree -from emf.model_merger.temporary_fixes import run_post_merge_processing, run_pre_merge_processing, \ - fix_model_outages, open_switches_in_network, fix_igm_ssh_vs_cgm_ssh_error + logger = logging.getLogger(__name__) parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.merger) @@ -45,20 +45,6 @@ def log_opdm_response(response): logger.debug(etree.tostring(response, pretty_print=True).decode()) -def convert_dict_str_to_bool(data_dict: dict): - for key, value in data_dict.items(): - if isinstance(value, str): - if value in ['True', 'true', 'TRUE']: - data_dict[key] = True - elif value in ['False', 'false', 'FALSE']: - data_dict[key] = False - elif isinstance(value, dict): - # Recursively converter nested dictionaries - data_dict[key] = convert_dict_str_to_bool(value) - - return data_dict - - @dataclass class MergedModel: network: pypowsybl.network = None @@ -91,6 +77,49 @@ def __init__(self): self.minio_service = minio_api.ObjectStorage() self.elk_logging_handler = get_elk_logging_handler() + @staticmethod + def run_loadflow(merged_model): + # Set starting point of lf settings priority list + if json.loads(ENABLE_DYNAMIC_MERGE_SETTINGS.lower()): + settings_list = [param.strip() for param in MERGE_LOAD_FLOW_SETTINGS_PRIORITY.split(",")] + settings_priority = next((i for i, value in enumerate(settings_list) if value == MERGE_LOAD_FLOW_SETTINGS), None) + settings_list = settings_list[settings_priority:] + else: + settings_list = [MERGE_LOAD_FLOW_SETTINGS] + + for lf_settings in settings_list: + logger.info(f"Solving loadflow with settings: {lf_settings}") + # report = pypowsybl.report.Reporter() + result = pypowsybl.loadflow.run_ac(network=merged_model.network, + parameters=getattr(loadflow_settings, lf_settings), + # reporter=loadflow_report, + ) + if result[0].status_text == 'Converged': + break + else: + logger.warning(f"Failed to solve loadflow with settings: {lf_settings}") + + result_dict = [attr_to_dict(island) for island in result] + # Modify all nested objects to native data types + for island in result_dict: + island['status'] = island['status'].name + # Extract only first slack bus from internal pypowsybl object + slack_bus_results = island.pop('slack_bus_results') + if slack_bus_results: + island['slack_bus_id'] = getattr(slack_bus_results[0], 'id', 'undefined') + island['active_power_mismatch'] = getattr(slack_bus_results[0], 'active_power_mismatch', float()) + else: + island['slack_bus_id'] = 'undefined' + island['active_power_mismatch'] = float() + + # merged_model.loadflow = json.loads(loadflow_report.to_json()) + # merged_model.loadflow = str(loadflow_report) + merged_model.loadflow = [island for island in result_dict if island['reference_bus_id']] + merged_model.loadflow_status = result[0].status.name # store main island loadflow status + merged_model.loadflow_settings = lf_settings + + return merged_model + def handle(self, task_object: dict, properties: dict, **kwargs): start_time = datetime.datetime.now(datetime.UTC) @@ -130,46 +159,41 @@ def handle(self, task_object: dict, properties: dict, **kwargs): mas = task_properties["mas"] version = task_properties["version"] model_replacement = task_properties["replacement"] - model_replacement_local = task_properties["replacement_local"] model_scaling = task_properties["scaling"] model_upload_to_opdm = task_properties["upload_to_opdm"] model_upload_to_minio = task_properties["upload_to_minio"] model_merge_report_send_to_elk = task_properties["send_merge_report"] - pre_temp_fixes = task_properties['pre_temp_fixes'] post_temp_fixes = task_properties['post_temp_fixes'] force_outage_fix = task_properties['force_outage_fix'] - open_non_retained_switches_between_tn = json.loads(OPEN_NON_RETAINED_SWITCHES_BETWEEN_TN.lower()) - try: - net_interchange_threshold = int(NET_INTERCHANGE_THRESHOLD) - except Exception: - net_interchange_threshold = 200 - - task_properties['net_interchange2_threshold'] = net_interchange_threshold - - remove_non_generators_from_slack_participation = True # Collect valid models from ObjectStorage - downloaded_models = get_latest_models_and_download(time_horizon, scenario_datetime, valid=False, data_source='OPDM') + downloaded_models = get_latest_models_and_download(time_horizon=time_horizon, + scenario_date=scenario_datetime, + valid=True, + data_source='OPDM') latest_boundary = get_latest_boundary() # Filter out models that are not to be used in merge - filtered_models = merge_functions.filter_models(downloaded_models, included_models, excluded_models, filter_on='pmd:TSO') + models = merge_functions.filter_models(models=downloaded_models, + included_models=included_models, + excluded_models=excluded_models, + filter_on='pmd:TSO') - # Get additional models directly from Minio + # Get additional models from ObjectStorage if local import is configured if local_import_models: - additional_models_data = get_latest_models_and_download(time_horizon=time_horizon, - scenario_date=scenario_datetime, - valid=True, - data_source='PDN') - additional_models_data = merge_functions.filter_models(models=additional_models_data, - included_models=local_import_models, - filter_on='pmd:TSO') + additional_models = get_latest_models_and_download(time_horizon=time_horizon, + scenario_date=scenario_datetime, + valid=True, + data_source='PDN') + additional_models = merge_functions.filter_models(models=additional_models, + included_models=local_import_models, + filter_on='pmd:TSO') - missing_local_import = [tso for tso in local_import_models if tso not in [model['pmd:TSO'] for model in additional_models_data]] - merged_model.excluded.extend([{'tso': tso, 'reason': 'Missing in PDN'} for tso in missing_local_import]) + missing_local_import = [tso for tso in local_import_models if tso not in [model['pmd:TSO'] for model in additional_models]] + merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-pdn'} for tso in missing_local_import]) # Perform local replacement if configured - if model_replacement_local and missing_local_import: + if model_replacement and missing_local_import: try: logger.info(f"Running replacement for local storage missing models: {missing_local_import}") replacement_models_local = run_replacement(tso_list=missing_local_import, @@ -182,33 +206,28 @@ def handle(self, task_object: dict, properties: dict, **kwargs): 'time_horizon': model['pmd:timeHorizon'], 'scenario_timestamp': model['pmd:scenarioDate']} for model in replacement_models_local] merged_model.replaced_entity.extend(replaced_entities_local) - - additional_models_data = additional_models_data + replacement_models_local - + additional_models.extend(replacement_models_local) except Exception as error: logger.error(f"Failed to run replacement: {error} {error.with_traceback()}") else: - additional_models_data = [] - - # Check model validity and availability - valid_models = [model for model in filtered_models if model['valid']] - invalid_models = [model['pmd:TSO'] for model in filtered_models if model not in valid_models] - if invalid_models: - merged_model.excluded.extend([{'tso': tso, 'reason': 'Invalid'} for tso in invalid_models]) + additional_models = [] # Check missing models for replacement if included_models: - missing_models = [model for model in included_models if model not in [model['pmd:TSO'] for model in valid_models]] + missing_models = [model for model in included_models if model not in [model['pmd:TSO'] for model in models]] if missing_models: - merged_model.excluded.extend([{'tso': tso, 'reason': 'Missing in OPDM'} for tso in missing_models]) + merged_model.excluded.extend([{'tso': tso, 'reason': 'missing-opdm'} for tso in missing_models]) else: if model_replacement: - available_tsos = get_available_tsos() - missing_models = [model for model in available_tsos if model not in [model['pmd:TSO'] for model in valid_models] + excluded_models] + # Get TSOs who models are available in storage for replacement period + available_tsos = get_tsos_available_in_storage(time_horizon=time_horizon) + valid_model_tsos = [model['pmd:TSO'] for model in models] + # Need to ensure that excluded models by task configuration would not be taken in replacement context + missing_models = [tso for tso in available_tsos if tso not in valid_model_tsos + excluded_models] else: missing_models = [] - # Run replacement on missing/invalid models + # Run replacement on missing models if model_replacement and missing_models: try: logger.info(f"Running replacement for missing models: {missing_models}") @@ -220,7 +239,7 @@ def handle(self, task_object: dict, properties: dict, **kwargs): 'scenario_timestamp': model['pmd:scenarioDate']} for model in replacement_models] merged_model.replaced_entity.extend(replaced_entities) - valid_models = valid_models + replacement_models + models.extend(replacement_models) merged_model.replaced = True else: merged_model.replaced = False @@ -228,64 +247,48 @@ def handle(self, task_object: dict, properties: dict, **kwargs): logger.error(f"Failed to run replacement: {error}") merged_model.replaced = False - # Store all relevant models for loading - valid_models = valid_models + additional_models_data - - # Return None if there are no models to be merged - if not valid_models: - logger.warning("Found no valid models to merge, returning None") - return None - - # Store models together with boundary - input_models = valid_models + [latest_boundary] + # Store models together with boundary set and check whether there are enough models to merge + input_models = models + additional_models + [latest_boundary] if len(input_models) < 2: - logger.warning("Found no models to merge, returning None") - return None - - # Run pre-processing - pre_p_start = datetime.datetime.now(datetime.UTC) - between_tn = pandas.DataFrame() - if pre_temp_fixes: - input_models, between_tn = run_pre_merge_processing(input_models, merging_area) - pre_p_end = datetime.datetime.now(datetime.UTC) - logger.debug(f"Pre-processing took: {(pre_p_end - pre_p_start).total_seconds()} seconds") + logger.warning("No valid models found for merging, exiting merge process") + properties.headers['success'] = False + return task_object, properties # Load network model and merge merge_start = datetime.datetime.now(datetime.UTC) - merged_model.network = merge_functions.load_model(opdm_objects=input_models) + merged_model.network = load_network_model(opdm_objects=input_models) merged_model.network_meta = attr_to_dict(instance=merged_model.network, sanitize_to_strings=True) - merged_model.included = [model['pmd:TSO'] for model in valid_models if model['pmd:TSO']] + merged_model.included = [model['pmd:TSO'] for model in input_models if model.get('pmd:TSO', None)] # Crosscheck replaced model outages with latest UAP if at least one Baltic model was replaced replaced_tso_list = [entity['tso'] for entity in merged_model.replaced_entity] - valid_tso_list = [tso['pmd:TSO'] for tso in valid_models] # Update model outages tso_list = [] - if force_outage_fix: #force outage fix on all models if set - tso_list = valid_tso_list - elif merging_area == 'BA' and any(tso in ['LITGRID', 'AST', 'ELERING'] for tso in replaced_tso_list): #by default do it on Baltic merge replaced models + if force_outage_fix: # force outage fix on all models if set + tso_list = merged_model.included + elif merging_area == 'BA' and any(tso in ['LITGRID', 'AST', 'ELERING'] for tso in replaced_tso_list): # by default do it on Baltic merge replaced models tso_list = replaced_tso_list - if tso_list: #if not set force and not replaced BA then nothing to fix - merged_model = fix_model_outages(merged_model=merged_model, - tso_list=tso_list, - scenario_datetime=scenario_datetime, - time_horizon=time_horizon) + if tso_list: # if not set force and not replaced BA then nothing to fix + merged_model = merge_functions.update_model_outages(merged_model=merged_model, + tso_list=tso_list, + scenario_datetime=scenario_datetime, + time_horizon=time_horizon) - # Various fixes from igmsshvscgmssh error - if remove_non_generators_from_slack_participation: - merged_model.network = fix_igm_ssh_vs_cgm_ssh_error(merged_model.network) + # Various corrections from igmsshvscgmssh error + if json.loads(REMOVE_GENERATORS_FROM_SLACK_DISTRIBUTION.lower()): + merged_model.network = handle_igm_ssh_vs_cgm_ssh_error(network_pre_instance=merged_model.network) - if open_non_retained_switches_between_tn and isinstance(between_tn, pandas.DataFrame): - if not between_tn.empty: - merged_model.network = open_switches_in_network(network_pre_instance=merged_model.network, - switches_dataframe=between_tn) + # Ensure boundary point EquivalentInjection are set to zero for paired tie lines + merged_model.network = merge_functions.ensure_paired_equivalent_injection_compatibility(network=merged_model.network) + + # Ensure boundary line connectivity consistency for paired boundary lines + merged_model.network = merge_functions.ensure_paired_boundary_line_connectivity(network=merged_model.network) # TODO - run other LF if default fails # Run loadflow on merged model - merged_model = merge_functions.run_lf(merged_model=merged_model, - loadflow_settings=getattr(loadflow_settings, MERGE_LOAD_FLOW_SETTINGS)) - logger.info(f"Loadflow status of main island: {merged_model.loadflow[0]['status_text']}") + merged_model = self.run_loadflow(merged_model=merged_model) + logger.info(f"Loadflow status of main island: {merged_model.loadflow_status} [settings: {merged_model.loadflow_settings}]") # Perform scaling if model_scaling: @@ -293,14 +296,10 @@ def handle(self, task_object: dict, properties: dict, **kwargs): # Set default time horizon and scenario timestamp if not provided if not schedule_time_horizon or schedule_time_horizon == "AUTO": schedule_time_horizon = time_horizon - else: - logger.warning(f"Using replaced schedules from timehorizon: {schedule_time_horizon}") - + if not schedule_start: schedule_start = scenario_datetime - else: - logger.warning(f"Replaced schedules from schedule_start: {schedule_start}") - + # Get aligned schedules ac_schedules = query_acnp_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) dc_schedules = query_hvdc_schedules(time_horizon=schedule_time_horizon, scenario_timestamp=schedule_start) @@ -311,35 +310,57 @@ def handle(self, task_object: dict, properties: dict, **kwargs): merged_model = scaler.scale_balance(model=merged_model, ac_schedules=ac_schedules, dc_schedules=dc_schedules, - lf_settings=getattr(loadflow_settings, MERGE_LOAD_FLOW_SETTINGS)) + lf_settings=getattr(loadflow_settings, merged_model.loadflow_settings)) except Exception as e: logger.error(e) merged_model.scaled = False else: - logger.warning(f"Schedule reference data not available: {schedule_time_horizon} for {schedule_start}, skipping model scaling") + logger.warning(f"Schedule reference data not available: {schedule_time_horizon} for {schedule_start}") + logger.warning(f"Network model schedule scaling not performed") merged_model.scaled = False # Record main merging process end merge_end = datetime.datetime.now(datetime.UTC) # Update time_horizon in case of generic ID process type - new_time_horizon = None if time_horizon.upper() == "ID": - time_horizon = merge_functions.calculate_intraday_time_horizon(scenario_datetime, task_creation_time) - new_time_horizon = time_horizon + time_horizon = merge_functions.set_intraday_time_horizon(scenario_datetime, task_creation_time) logger.info(f"Setting intraday time horizon to: {time_horizon}") + # Set merged model name + model_type = "RMM" if merging_area == "BA" else "CGM" + merged_model.name = f"{model_type}_{time_horizon}_{version}_{parse_datetime(scenario_datetime):%Y%m%dT%H%MZ}_{merging_area}_{uuid4()}" + + # Crate OPDM object for merged model + opdm_object_meta = merge_functions.create_merged_model_opdm_object( + object_id=merged_model.network_meta['id'].split("uuid:")[-1], + time_horizon=time_horizon, + merging_entity=merging_entity, + merging_area=merging_area, + scenario_date=scenario_datetime, + mas=mas, + version=version, + ) + # Export merged model + # TODO change here to export SSH profiles as well + exported_model = merge_functions.export_merged_model(network=merged_model.network, + opdm_object_meta=opdm_object_meta, + profiles=["SV"], + cgm_convention=False) + # Run post-processing post_p_start = datetime.datetime.now(datetime.UTC) - sv_data, ssh_data = run_post_merge_processing(input_models=input_models, - merged_model=merged_model, - task_properties=task_properties, - small_island_size=SMALL_ISLAND_SIZE, - enable_temp_fixes=post_temp_fixes, - time_horizon=new_time_horizon) + logger.info(f"Starting merged model post-processing") + # TODO here should be one existing network structure. IIDM model can be exported and removed to release memory + sv_data, ssh_data, opdm_object_meta = merge_functions.run_post_merge_processing(input_models=input_models, + exported_model=exported_model, + opdm_object_meta=opdm_object_meta, + enable_temp_fixes=post_temp_fixes, + task_properties=task_properties + ) # Package both input models and exported CGM profiles to in memory zip files - serialized_data = merge_functions.export_to_cgmes_zip([ssh_data, sv_data]) + serialized_data = export_to_cgmes_zip([ssh_data, sv_data]) post_p_end = datetime.datetime.now(datetime.UTC) logger.debug(f"Post processing took: {(post_p_end - post_p_start).total_seconds()} seconds") logger.debug(f"Merging took: {(merge_end - merge_start).total_seconds()} seconds") @@ -354,16 +375,12 @@ def handle(self, task_object: dict, properties: dict, **kwargs): callback=log_opdm_response, file_path_or_file_object=item) merged_model.uploaded_to_opde = True - except: - logging.error(f"""Unexpected error on uploading to OPDM:""", exc_info=True) + except Exception as error: + logging.error(f"Unexpected error on uploading to OPDM: {error}", exc_info=True) # Create zipped model data - if merging_area == 'BA': - merged_model.name = f"RMM_{time_horizon}_{version}_{parse_datetime(scenario_datetime):%Y%m%dT%H%MZ}_{merging_area}_{uuid4()}" - else: - merged_model.name = f"CGM_{time_horizon}_{version}_{parse_datetime(scenario_datetime):%Y%m%dT%H%MZ}_{merging_area}_{uuid4()}" - merged_model_data = BytesIO() - with ZipFile(merged_model_data, "w") as merged_model_zip: + merged_model_object = BytesIO() + with ZipFile(merged_model_object, "w") as merged_model_zip: # Include CGM model files for item in serialized_data: merged_model_zip.writestr(item.name, item.getvalue()) @@ -371,12 +388,10 @@ def handle(self, task_object: dict, properties: dict, **kwargs): # Include original IGM files for input_model in input_models: for instance in input_model['opde:Component']: - if instance['opdm:Profile']['pmd:cgmesProfile'] in ['EQ', 'TP', 'EQBD', 'TPBD']: - file_object = opdmprofile_to_bytes(instance) + if instance['opdm:Profile']['pmd:cgmesProfile'] in ['EQ', 'TP', 'EQBD', 'TPBD', 'EQ_BD', 'TP_BD']: + file_object = get_opdm_component_data_bytes(opdm_component=instance) logging.info(f"Adding file: {file_object.name}") merged_model_zip.writestr(file_object.name, file_object.getvalue()) - - merged_model_object = merged_model_data merged_model_object.name = f"{OUTPUT_MINIO_FOLDER}/{merged_model.name}.zip" # Upload to Minio storage @@ -384,11 +399,14 @@ def handle(self, task_object: dict, properties: dict, **kwargs): logger.info(f"Uploading merged model to MINIO: {merged_model_object.name}") minio_metadata = merge_functions.evaluate_trustability(merged_model.__dict__, task['task_properties']) try: - response = self.minio_service.upload_object(merged_model_object, bucket_name=OUTPUT_MINIO_BUCKET, metadata=minio_metadata) + response = self.minio_service.upload_object(file_path_or_file_object=merged_model_object, + bucket_name=OUTPUT_MINIO_BUCKET, + metadata=minio_metadata, + ) if response: merged_model.uploaded_to_minio = True - except: - logging.error(f"""Unexpected error on uploading to Object Storage:""", exc_info=True) + except Exception as error: + logging.error(f"Unexpected error on uploading to Object Storage: {error}", exc_info=True) logger.info(f"Merged model creation done for: {merged_model.name}") @@ -404,29 +422,35 @@ def handle(self, task_object: dict, properties: dict, **kwargs): logger.debug(task) # Update merged model attributes - merged_model.loadflow_settings = MERGE_LOAD_FLOW_SETTINGS merged_model.duration_s = (end_time - merge_start).total_seconds() merged_model.content_reference = merged_model_object.name - # Send merge report to Elastic + # Update OPDM object data with CGM relevant data and send to Elastic + opdm_object_meta['pmd:content-reference'] = merged_model.content_reference + response = elastic.Elastic.send_to_elastic(index=OPDE_MODELS_ELK_INDEX, json_message=opdm_object_meta) + + # Send merge report and opdm object metadata to Elastic if model_merge_report_send_to_elk: logger.info(f"Sending merge report to Elastic") try: merge_report = merge_functions.generate_merge_report(merged_model=merged_model, task=task) try: - response = elastic.Elastic.send_to_elastic(index=MERGE_REPORT_ELK_INDEX, - json_message=merge_report) + response = elastic.Elastic.send_to_elastic(index=MERGE_REPORT_ELK_INDEX, json_message=merge_report) except Exception as error: logger.error(f"Merge report sending to Elastic failed: {error}") except Exception as error: logger.error(f"Failed to create merge report: {error}") + # Append message headers with OPDM root metadata + extracted_meta = {key: value for key, value in opdm_object_meta.items() if isinstance(value, str)} + properties.headers = extracted_meta + # Stop Trace self.elk_logging_handler.stop_trace() logger.info(f"Merge task finished for model: {merged_model.name}") - return task, properties + return json.dumps(opdm_object_meta), properties if __name__ == "__main__": @@ -459,23 +483,21 @@ def handle(self, task_object: dict, properties: dict, **kwargs): "job_period_start": "2024-05-24T22:00:00+00:00", "job_period_end": "2024-05-25T06:00:00+00:00", "task_properties": { - "timestamp_utc": "2025-04-29T22:30:00+00:00", + "timestamp_utc": "2025-07-06T12:30:00+00:00", "merge_type": "BA", "merging_entity": "BALTICRCC", - "included": ["AST"], + "included": ["LITGRID", "AST", "ELERING", "PSE"], "excluded": [], - "local_import": ["LITGRID"], - "time_horizon": "ID", - "version": "00", + "local_import": [], + "time_horizon": "1D", + "version": "001", "mas": "http://www.baltic-rsc.eu/OperationalPlanning", - "pre_temp_fixes": "True", "post_temp_fixes": "True", "fix_net_interchange2": "True", "replacement": "True", - "replacement_local": "True", "scaling": "True", "upload_to_opdm": "False", - "upload_to_minio": "False", + "upload_to_minio": "True", "send_merge_report": "False", "force_outage_fix": "False", } diff --git a/emf/model_merger/replacement.py b/emf/model_merger/replacement.py index e14000d5..0a6ead95 100644 --- a/emf/model_merger/replacement.py +++ b/emf/model_merger/replacement.py @@ -6,19 +6,20 @@ import json from dateutil import parser from pathlib import Path -from emf.common.integrations.object_storage.models import query_data, get_content +from emf.common.integrations.object_storage.models import query_data, get_content, fetch_unique_values from emf.common.integrations.minio_api import * from emf.common.config_parser import parse_app_properties logger = logging.getLogger(__name__) parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.replacement) +replacement_config = json.load(config.paths.cgm_worker.replacement_conf) def run_replacement(tso_list: list, time_horizon: str, scenario_date: str, - config: list = json.load(config.paths.cgm_worker.replacement_conf), + config: list = replacement_config, data_source: str = 'OPDM', ): """ @@ -35,9 +36,11 @@ def run_replacement(tso_list: list, replacements = pd.DataFrame() # TODO time horizon exclusion logic + exclude available models from query # TODO put in query object type if CGM metadata objects will be stored + # Get replacement length by time horizon + query_filter = 'now-' + config["replacement_length"]["request_list"][config["time_horizon"]["request_list"].index(time_horizon)] + # Query for available replacement models query = {"pmd:TSO.keyword": tso_list, "valid": True, "data-source": data_source} - body = query_data(query, QUERY_FILTER) - model_df = pd.DataFrame(body) + model_df = pd.DataFrame(query_data(query, query_filter)) # Set scenario dat to UTC if not model_df.empty: @@ -64,13 +67,13 @@ def run_replacement(tso_list: list, replaced_tso = replacements['pmd:TSO'].unique().tolist() not_replaced = [model for model in unique_tsos_list if model not in replaced_tso] if not_replaced: - logger.error(f"Unable to find replacements within given replacement logic for TSO's: {not_replaced}") + logger.warning(f"Unable to find replacements within given replacement logic for TSO's: {not_replaced}") tso_missing = [model for model in tso_list if model not in unique_tsos_list] if tso_missing: logger.info(f"No replacement models found for TSO(s): {tso_missing}") else: - logger.error(f"No replacement models found, replacement list is empty") + logger.warning(f"No replacement models found, replacement list is empty") else: logger.info(f"No replacement models found in Elastic for TSO(s): {tso_list}") @@ -154,7 +157,7 @@ def make_lists_priority(timestamp, target_timehorizon, conf): hour_list_final = list(map(lambda x: (date_time + parse_duration(x)).strftime("%H:%M"), hour_list)) day_list_final = list(map(lambda x: (date_time + parse_duration(x)).strftime("%Y-%m-%d"), day_list)) - business_list = conf["timeHorizon"]["Request_list"] + business_list = conf["time_horizon"]["request_list"] business_list_final = business_list[business_list.index(target_timehorizon):] # make list of relevant businesstypes # Month ahead requires separate replacement logic @@ -196,11 +199,13 @@ def create_replacement_table(target_timestamp, target_timehorizon, valid_models_ return valid_models_df -def get_available_tsos(): - query = {"opde:Object-Type": "IGM", "valid": True} - body = query_data(query, QUERY_FILTER) - key = 'pmd:TSO' - return list({item[key] for item in body if key in item}) +def get_tsos_available_in_storage(time_horizon: str): + metadata = {"opde:Object-Type": "IGM", "valid": True} + # Get query length by time horizon from configuration + query_filter = 'now-' + replacement_config["replacement_length"]["request_list"][replacement_config["time_horizon"]["request_list"].index(time_horizon)] + unique_tsos = fetch_unique_values(metadata_query=metadata, field="pmd:TSO.keyword", query_filter=query_filter) + + return unique_tsos def get_first_monday_of_last_month(timestamp): diff --git a/emf/model_merger/scaler.py b/emf/model_merger/scaler.py index cc7c44ab..6d80268d 100644 --- a/emf/model_merger/scaler.py +++ b/emf/model_merger/scaler.py @@ -22,6 +22,13 @@ scheduling areas are below the discrepancy thresholds, as defined previously; " In any case after the 15th iteration16 (adjustments take place within the iterations). """ + +""" +NOTES: + - power factor sign defines whether P and Q values has opposite sign. This needs to be ensured because new Q values +are calculated from P values, then the power factors sign defines what sign should be for new Q value. +""" + import pypowsybl as pp import logging import pandas as pd @@ -31,8 +38,8 @@ from emf.common.config_parser import parse_app_properties from emf.common.decorators import performance_counter from emf.common.integrations.object_storage.schedules import query_acnp_schedules, query_hvdc_schedules -from emf.common.loadflow_tool.helper import attr_to_dict, get_network_elements, get_slack_generators, \ - get_connected_components_data +from emf.common.helpers.utils import attr_to_dict +from emf.common.helpers.loadflow import get_network_elements, get_slack_generators, get_connected_components_data from emf.common.loadflow_tool.loadflow_settings import CGM_RELAXED_1 logger = logging.getLogger(__name__) @@ -110,15 +117,21 @@ def scale_balance(model: object, # Get pre-scale HVDC setpoints dangling_lines = get_network_elements(network, pp.network.ElementType.DANGLING_LINE, all_attributes=True) + dangling_lines['power_factor'] = dangling_lines['boundary_q'] / dangling_lines['boundary_p'] # estimate the power factor + dangling_lines['power_factor'] = dangling_lines['power_factor'].fillna(0) # handle zero division dangling_lines['boundary_p'] = dangling_lines['boundary_p'] * -1 # invert boundary_p sign to match flow direction - prescale_hvdc_sp = dangling_lines[dangling_lines.isHvdc == 'true'][['lineEnergyIdentificationCodeEIC', 'boundary_p']] - prescale_hvdc_sp = prescale_hvdc_sp.rename(columns={'boundary_p': 'value'}) + dangling_lines['boundary_q'] = dangling_lines['boundary_q'] * -1 # invert boundary_q sign to match flow direction + prescale_hvdc_sp = dangling_lines[dangling_lines.isHvdc == 'true'][['lineEnergyIdentificationCodeEIC', 'boundary_p', 'boundary_q']] + prescale_hvdc_sp = prescale_hvdc_sp.rename(columns={'boundary_p': 'value', 'boundary_q': 'value_q'}) _hvdc_results.append(pd.concat([prescale_hvdc_sp.set_index('lineEnergyIdentificationCodeEIC').value, pd.Series({'KEY': 'prescale-setpoint'})]).to_dict()) - for dclink in prescale_hvdc_sp.to_dict('records'): - logger.info(f"[INITIAL] PRE-SCALE HVDC setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value'], 2)} MW") + for dclink in prescale_hvdc_sp.sort_values('lineEnergyIdentificationCodeEIC').to_dict('records'): + logger.info(f"[INITIAL] PRE-SCALE HVDC active power setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value'], 2)} MW") + logger.debug(f"[INITIAL] PRE-SCALE HVDC reactive power setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value_q'], 2)} MW") + # Mapping HVDC schedules to network - scalable_hvdc = dangling_lines[dangling_lines.isHvdc == 'true'][['lineEnergyIdentificationCodeEIC', _country_col, 'ucte_xnode_code']] + _cols_to_keep = ['lineEnergyIdentificationCodeEIC', _country_col, 'ucte_xnode_code', 'power_factor'] + scalable_hvdc = dangling_lines[dangling_lines.isHvdc == 'true'][_cols_to_keep] scalable_hvdc.reset_index(inplace=True) scalable_hvdc = scalable_hvdc.merge(target_hvdc_sp_df, left_on='lineEnergyIdentificationCodeEIC', right_on='registered_resource') mask = (scalable_hvdc[_country_col] == scalable_hvdc['in_domain']) | (scalable_hvdc[_country_col] == scalable_hvdc['out_domain']) @@ -132,13 +145,15 @@ def scale_balance(model: object, scalable_hvdc = scalable_hvdc.set_index('id') # Updating HVDC network elements to scheduled values - scalable_hvdc_target = scalable_hvdc[['value', 'lineEnergyIdentificationCodeEIC']] - network.update_dangling_lines(id=scalable_hvdc_target.index, p0=scalable_hvdc_target.value) + scalable_hvdc_target = scalable_hvdc[['value', 'lineEnergyIdentificationCodeEIC', 'power_factor']] + scalable_hvdc_target['value_q'] = scalable_hvdc_target.value * scalable_hvdc_target.power_factor # ensure power factor is kept + network.update_dangling_lines(id=scalable_hvdc_target.index, p0=scalable_hvdc_target.value, q0=scalable_hvdc_target.value_q) _hvdc_results.append(pd.concat([scalable_hvdc_target.set_index('lineEnergyIdentificationCodeEIC').value, pd.Series({'KEY': 'postscale-setpoint'})]).to_dict()) logger.info(f"[INITIAL] HVDC elements updated to target values: {scalable_hvdc_target['lineEnergyIdentificationCodeEIC'].values}") - for dclink in scalable_hvdc_target.to_dict('records'): - logger.info(f"[INITIAL] POST-SCALE HVDC setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value'], 2)} MW") + for dclink in scalable_hvdc_target.sort_values('lineEnergyIdentificationCodeEIC').to_dict('records'): + logger.info(f"[INITIAL] POST-SCALE HVDC active power setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value'], 2)} MW") + logger.debug(f"[INITIAL] POST-SCALE HVDC reactive power setpoint of {dclink['lineEnergyIdentificationCodeEIC']}: {round(dclink['value_q'], 2)} MVar") # Get AC net positions scaling perimeter -> non-negative ConformLoads loads = get_network_elements(network, pp.network.ElementType.LOAD, all_attributes=True) @@ -194,6 +209,8 @@ def scale_balance(model: object, # Get pre-scale total network balance by each component -> AC+DC net position dangling_lines = get_network_elements(network, pp.network.ElementType.DANGLING_LINE, all_attributes=True) + dangling_lines['power_factor'] = dangling_lines['boundary_q'] / dangling_lines['boundary_p'] # estimate the power + dangling_lines['power_factor'] = dangling_lines['power_factor'].fillna(0) # handle zero division dangling_lines['boundary_p'] = dangling_lines['boundary_p'] * -1 # invert boundary_p sign to match flow direction prescale_network_np = {k: round(dangling_lines[dangling_lines.country.isin(v['countries'])].boundary_p.sum()) for k, v in converged_components.items()} _scaling_results.append({'KEY': 'prescale-network-np', 'GLOBAL': prescale_network_np, 'ITER': _iteration}) @@ -221,7 +238,8 @@ def scale_balance(model: object, prescale_network_acnp_target.dropna(inplace=True) logger.info(f"[ITER {_iteration}] Scaling network component {component_key} {v['countries']} ACNP to scheduled: {scheduled_component_acnp}") network.update_dangling_lines(id=prescale_network_acnp_target.index, - p0=prescale_network_acnp_target.to_list()) # TODO maintain power factor + p0=prescale_network_acnp_target.to_list(), + q0=(prescale_network_acnp_target * relevant_dangling_lines.power_factor).to_list()) _scaling_results.append({'KEY': 'target-network-acnp', 'GLOBAL': target_network_acnp, 'ITER': _iteration}) # Solving loadflow after aligning total network AC net position to scheduled @@ -235,6 +253,7 @@ def scale_balance(model: object, # TODO need to consider how to evaluate it in case of multiple islands. For example if one of the island diverges but not the main if not validate_loadflow_status(results=pf_results): model.scaled = False + logger.warning(f"Terminating network scaling due to divergence in main island after island ACNP alignment") return model # Validate total network AC net position alignment diff --git a/emf/model_merger/temporary.py b/emf/model_merger/temporary.py new file mode 100644 index 00000000..0fd162a2 --- /dev/null +++ b/emf/model_merger/temporary.py @@ -0,0 +1,338 @@ +import pypowsybl +import triplets +import pandas as pd +import logging +import uuid +from decimal import Decimal +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets + +logger = logging.getLogger(__name__) + + +def remove_small_islands(solved_data, island_size_limit): + # TODO - EVALUATE LEGACY + small_island = pd.DataFrame(solved_data.query("KEY == 'TopologicalIsland.TopologicalNodes'").ID.value_counts()).reset_index().query("count <= @island_size_limit") + solved_data = triplets.rdf_parser.remove_triplet_from_triplet(solved_data, small_island, columns=["ID"]) + logger.info(f"Removed {len(small_island)} island(s) with size <= {island_size_limit}") + return solved_data + + +def remove_equivalent_shunt_section(sv_data: pd.DataFrame, models_as_triplets: pd.DataFrame): + # TODO - EVALUATE LEGACY + """Remove Shunt Sections for EQV Shunts from SV profile""" + + equiv_shunt = models_as_triplets.query("KEY == 'Type' and VALUE == 'EquivalentShunt'") + if len(equiv_shunt) > 0: + shunts_to_remove = sv_data.merge( + sv_data.query("KEY == 'SvShuntCompensatorSections.ShuntCompensator'").merge(equiv_shunt.ID, left_on='VALUE', + right_on="ID", how='inner', + suffixes=('', '_EQVShunt')).ID) + if len(shunts_to_remove) > 0: + logger.warning(f'Removing invalid SvShuntCompensatorSections for EquivalentShunt') + sv_data = triplets.rdf_parser.remove_triplet_from_triplet(sv_data, shunts_to_remove) + + return sv_data + + +def add_missing_sv_tap_steps(sv_data: pd.DataFrame, ssh_data: pd.DataFrame): + # TODO - EVALUATE LEGACY + """Update missing tap changer tap steps in SV""" + + ssh_tap_steps = ssh_data.query("KEY == 'TapChanger.step'") + sv_tap_steps = sv_data.query("KEY == 'SvTapStep.TapChanger'") + + missing_sv_tap_steps = ssh_tap_steps.merge(sv_tap_steps[['VALUE']], + left_on='ID', + right_on="VALUE", + how='left', + indicator=True, + suffixes=('', '_SV')).query("_merge == 'left_only'") + + tap_steps_to_be_added = [] + SV_INSTANCE_ID = sv_data.INSTANCE_ID.iloc[0] + for tap_changer in missing_sv_tap_steps.itertuples(): + ID = str(uuid.uuid4()) + logger.warning(f'Missing SvTapStep for {tap_changer.ID}, adding SvTapStep {ID} and taking tap value {tap_changer.VALUE} from SSH') + tap_steps_to_be_added.extend([ + (ID, 'Type', 'SvTapStep', SV_INSTANCE_ID), + (ID, 'SvTapStep.TapChanger', tap_changer.ID, SV_INSTANCE_ID), + (ID, 'SvTapStep.position', tap_changer.VALUE, SV_INSTANCE_ID), + ]) + + sv_data = pd.concat([sv_data, pd.DataFrame(tap_steps_to_be_added, columns=['ID', 'KEY', 'VALUE', 'INSTANCE_ID'])], ignore_index=True) + + return sv_data + + +def open_switches_in_network(network_pre_instance: pypowsybl.network.Network, switches_dataframe: pd.DataFrame): + # TODO - EVALUATE LEGACY + """ + Opens switches in loaded network given by dataframe (uses ID for merging) + :param network_pre_instance: pypowsybl Network instance where igms are loaded in + :param switches_dataframe: dataframe + """ + logger.info(f"Opening {len(switches_dataframe.index)} switches") + switches = network_pre_instance.get_switches(all_attributes=True).reset_index() + switches = switches.merge(switches_dataframe[['ID']].rename(columns={'ID': 'id'}), on='id') + non_retained_closed = switches.merge(switches_dataframe.rename(columns={'ID': 'id'}), on='id')[['id', 'open']] + non_retained_closed['open'] = True + network_pre_instance.update_switches(non_retained_closed.set_index('id')) + return network_pre_instance + + +def check_and_fix_dependencies(cgm_sv_data, cgm_ssh_data, original_data): + # TODO - EVALUATE LEGACY + """ + Seems that pypowsybl ver 1.6.0 managed to get rid of dependencies in exported file. This gathers them from + SSH profiles and from the original models + :param cgm_sv_data: merged SV profile that is missing the dependencies + :param cgm_ssh_data: merged SSH profiles, will be used to get SSH dependencies + :param original_data: original models, will be used to get TP dependencies + :return updated merged SV profile + """ + + # some_data = load_opdm_objects_to_triplets(opdm_objects=original_data) + some_data = get_opdm_data_from_models(model_data=original_data) + tp_file_ids = some_data[(some_data['KEY'] == 'Model.profile') & (some_data['VALUE'].str.contains('Topology'))] + + ssh_file_ids = cgm_ssh_data[(cgm_ssh_data['KEY'] == 'Model.profile') & + (cgm_ssh_data['VALUE'].str.contains('SteadyStateHypothesis'))] + dependencies = pd.concat([tp_file_ids, ssh_file_ids], ignore_index=True, sort=False) + existing_dependencies = cgm_sv_data[cgm_sv_data['KEY'] == 'Model.DependentOn'] + dependency_difference = existing_dependencies.merge(dependencies[['ID']].rename(columns={'ID': 'VALUE'}), + on='VALUE', how='outer', indicator=True) + if not dependency_difference.query('_merge == "right_only"').empty: + cgm_sv_data = triplets.rdf_parser.remove_triplet_from_triplet(cgm_sv_data, existing_dependencies) + full_model_id = cgm_sv_data[(cgm_sv_data['KEY'] == 'Type') & (cgm_sv_data['VALUE'] == 'FullModel')] + dependencies_to_update = dependency_difference.query('_merge != "left_only"') + logger.warning(f"Mismatch of dependencies. Inserting {len(dependencies_to_update.index)} dependencies to SV profile") + new_dependencies = dependencies_to_update[['VALUE']].copy().reset_index(drop=True) + new_dependencies.loc[:, 'KEY'] = 'Model.DependentOn' + new_dependencies.loc[:, 'ID'] = full_model_id['ID'].iloc[0] + new_dependencies.loc[:, 'INSTANCE_ID'] = full_model_id['INSTANCE_ID'].iloc[0] + cgm_sv_data = triplets.rdf_parser.update_triplet_from_triplet(cgm_sv_data, new_dependencies) + return cgm_sv_data + + +def handle_igm_ssh_vs_cgm_ssh_error(network_pre_instance: pypowsybl.network.Network): + # TODO - EVALUATE LEGACY + """ + Implements various fixes to suppress igm ssh vs cgm ssh error + 1) Get all generators and remove them from slack distribution + 2) If generators have target_p outside the endpoints ('limits') of a curve then set it to be within + 3) Condensers p should not be modified so if it is not 0 then it sets the target_p to equal the existing p + :param network_pre_instance: pypowsybl Network instance where igms are loaded in + :return updated network_pre_instance + """ + try: + all_generators = network_pre_instance.get_elements(element_type=pypowsybl.network.ElementType.GENERATOR, + all_attributes=True).reset_index() + generators_mask = (all_generators['CGMES.synchronousMachineOperatingMode'].str.contains('generator')) + not_generators = all_generators[~generators_mask] + generators = all_generators[generators_mask] + curve_points = (network_pre_instance + .get_elements(element_type=pypowsybl.network.ElementType.REACTIVE_CAPABILITY_CURVE_POINT, + all_attributes=True).reset_index()) + curve_limits = (curve_points.merge(generators[['id']], on='id') + .groupby('id').agg(curve_p_min=('p', 'min'), curve_p_max=('p', 'max'))).reset_index() + curve_generators = generators.merge(curve_limits, on='id') + # low end can be zero + curve_generators = curve_generators[(curve_generators['target_p'] > curve_generators['curve_p_max']) | + ((curve_generators['target_p'] > 0) & + (curve_generators['target_p'] < curve_generators['curve_p_min']))] + if not curve_generators.empty: + logger.warning(f"Found {len(curve_generators.index)} generators for " + f"which p > max(reactive capacity curve(p)) or p < min(reactive capacity curve(p))") + + # Solution 1: set max_p from curve max, it should contain p on target-p + upper_limit_violated = curve_generators[(curve_generators['max_p'] > curve_generators['curve_p_max'])] + if not upper_limit_violated.empty: + logger.warning(f"Updating max p from curve for {len(upper_limit_violated.index)} generators") + upper_limit_violated['max_p'] = upper_limit_violated['curve_p_max'] + network_pre_instance.update_generators(upper_limit_violated[['id', 'max_p']].set_index('id')) + + lower_limit_violated = curve_generators[(curve_generators['min_p'] < curve_generators['curve_p_min'])] + if not lower_limit_violated.empty: + logger.warning(f"Updating min p from curve for {len(lower_limit_violated.index)} generators") + lower_limit_violated.loc[:, 'min_p'] = lower_limit_violated['curve_p_min'] + network_pre_instance.update_generators(lower_limit_violated[['id', 'min_p']].set_index('id')) + + # Solution 2: discard generator from participating + extensions = network_pre_instance.get_extensions('activePowerControl') + remove_curve_generators = extensions.merge(curve_generators[['id']], + left_index=True, right_on='id') + if not remove_curve_generators.empty: + remove_curve_generators['participate'] = False + network_pre_instance.update_extensions('activePowerControl', + remove_curve_generators.set_index('id')) + condensers = all_generators[(all_generators['CGMES.synchronousMachineType'].str.contains('condenser')) + & (abs(all_generators['p']) > 0) + & (abs(all_generators['target_p']) == 0)] + # Fix condensers that have p not zero by setting their target_p to equal to p + if not condensers.empty: + logger.warning(f"Found {len(condensers.index)} condensers for which p ~= 0 & target_p = 0") + condensers.loc[:, 'target_p'] = condensers['p'] * (-1) + network_pre_instance.update_generators(condensers[['id', 'target_p']].set_index('id')) + # Remove all not generators from active power distribution + if not not_generators.empty: + logger.warning(f"Removing {len(not_generators.index)} machines from power distribution") + extensions = network_pre_instance.get_extensions('activePowerControl') + remove_not_generators = extensions.merge(not_generators[['id']], left_index=True, right_on='id') + remove_not_generators['participate'] = False + remove_not_generators = remove_not_generators.set_index('id') + network_pre_instance.update_extensions('activePowerControl', remove_not_generators) + except Exception as ex: + logger.warning(f"Unable to pre-process for igm-cgm-ssh error: {ex}") + + return network_pre_instance + + +def take_best_match_for_sv_voltage(input_data, column_name: str = 'v', to_keep: bool = True): + # TODO - EVALUATE LEGACY + """ + Returns one row for with sv voltage id for topological node + 1) Take the first + 2) If first is zero take first non-zero row if exists + :param input_data: input dataframe + :param column_name: name of the column + :param to_keep: either to keep or discard a value + """ + first_row = input_data.iloc[0] + if to_keep: + remaining_rows = input_data[input_data[column_name] != 0] + if first_row[column_name] == 0 and not remaining_rows.empty: + first_row = remaining_rows.iloc[0] + else: + remaining_rows = input_data[input_data[column_name] == 0] + if first_row[column_name] != 0 and not remaining_rows.empty: + first_row = remaining_rows.iloc[0] + return first_row + + +def get_opdm_data_from_models(model_data: list | pd.DataFrame): + """ + Check if input is already parsed to triplets. Do it otherwise + :param model_data: input models + :return triplets + """ + if not isinstance(model_data, pd.DataFrame): + model_data = load_opdm_objects_to_triplets(model_data) + return model_data + + +def get_boundary_nodes_between_igms(model_data: list | pd.DataFrame): + # TODO - EVALUATE LEGACY + """ + Filters out nodes that are between the igms (mentioned at least 2 igms) + :param model_data: input models + : return series of node ids + """ + model_data = get_opdm_data_from_models(model_data=model_data) + all_boundary_nodes = model_data[(model_data['KEY'] == 'TopologicalNode.boundaryPoint') & + (model_data['VALUE'] == 'true')] + # Get boundary nodes that exist in igms + merged = pd.merge(all_boundary_nodes, + model_data[(model_data['KEY'] == 'SvVoltage.TopologicalNode')], + left_on='ID', right_on='VALUE', suffixes=('_y', '')) + # Get duplicates (all of them) then duplicated values. keep=False marks all duplicates True, 'first' marks first + # occurrence to false, 'last' marks last occurrence to false. If any of them is used then in case duplicates are 2 + # then 1 is retrieved, if duplicates >3 then duplicates-1 retrieved. So, get all the duplicates and as a second + # step, drop the duplicates + merged = (merged[merged.duplicated(['VALUE'], keep=False)]).drop_duplicates(subset=['VALUE']) + in_several_igms = (merged["VALUE"]).to_frame().rename(columns={'VALUE': 'ID'}) + return in_several_igms + + +def remove_duplicate_sv_voltages(cgm_sv_data, original_data): + # TODO - EVALUATE LEGACY + """ + Pypowsybl 1.6.0 provides multiple sets of SvVoltage values for the topological nodes that are boundary nodes (from + each IGM side that uses the corresponding boundary node). So this is a hack that removes one of them (preferably the + one that is zero). + :param cgm_sv_data: merged SV profile from where duplicate SvVoltage values are removed + :param original_data: will be used to get boundary node ids + :return updated merged SV profile + """ + # Check that models are in triplets + some_data = get_opdm_data_from_models(model_data=original_data) + # Get ids of boundary nodes that are shared by several igms + in_several_igms = (get_boundary_nodes_between_igms(model_data=some_data)) + # Get SvVoltage Ids corresponding to shared boundary nodes + sv_voltage_ids = pd.merge(cgm_sv_data[cgm_sv_data['KEY'] == 'SvVoltage.TopologicalNode'], + in_several_igms.rename(columns={'ID': 'VALUE'}), on='VALUE') + # Get SvVoltage voltage values for corresponding SvVoltage Ids + sv_voltage_values = pd.merge(cgm_sv_data[cgm_sv_data['KEY'] == 'SvVoltage.v'][['ID', 'VALUE']]. + rename(columns={'VALUE': 'SvVoltage.v'}), + sv_voltage_ids[['ID', 'VALUE']]. + rename(columns={'VALUE': 'SvVoltage.SvTopologicalNode'}), on='ID') + # Just in case convert the values to numeric + sv_voltage_values[['SvVoltage.v']] = (sv_voltage_values[['SvVoltage.v']].apply(lambda x: x.apply(Decimal))) + # Group by topological node id and by some logic take SvVoltage that will be dropped + voltages_to_keep = (sv_voltage_values.groupby(['SvVoltage.SvTopologicalNode']). + apply(lambda x: take_best_match_for_sv_voltage(input_data=x, + column_name='SvVoltage.v', + to_keep=True), include_groups=False)) + voltages_to_discard = sv_voltage_values.merge(voltages_to_keep['ID'], on='ID', how='left', indicator=True) + voltages_to_discard = voltages_to_discard[voltages_to_discard['_merge'] == 'left_only'] + if not voltages_to_discard.empty: + logger.info(f"Removing {len(voltages_to_discard.index)} duplicate voltage levels from boundary nodes") + sv_voltages_to_remove = pd.merge(cgm_sv_data, voltages_to_discard['ID'].to_frame(), on='ID') + cgm_sv_data = triplets.rdf_parser.remove_triplet_from_triplet(cgm_sv_data, sv_voltages_to_remove) + + return cgm_sv_data + + +def set_paired_boundary_injections_to_zero(original_models, cgm_ssh_data): + """Where there are paired boundary points, equivalent injections need to be modified + Set P and Q to 0 - so that no additional consumption or production is on tie line + Set voltage control off - so that no additional consumption or production is on tie line + Set terminal to connected - to be sure we have paired connected injections at boundary point + In some models terminals are missing references to ConnectivityNodes + """ + + topological_boundary_points = original_models.query("KEY == 'TopologicalNode.boundaryPoint' and VALUE == 'true'")[["ID"]] + try: + terminals = original_models.type_tableview("Terminal").reset_index()[['ID', + 'Terminal.ConductingEquipment', + 'Terminal.ConnectivityNode', + 'Terminal.TopologicalNode']] + except KeyError: + terminals = original_models.type_tableview("Terminal").reset_index()[['ID', + 'Terminal.ConductingEquipment', + 'Terminal.TopologicalNode']] + injections = cgm_ssh_data.type_tableview('EquivalentInjection').reset_index()[['ID', + # 'EquivalentInjection.p', + # 'EquivalentInjection.q', + # 'EquivalentInjection.regulationStatus' + ]] + topological_boundary_points = topological_boundary_points.merge(terminals, + left_on="ID", + right_on="Terminal.TopologicalNode", + suffixes=('_TopologicalNode', '_Terminal')) + topological_injections = injections.merge(topological_boundary_points, + left_on="ID", + right_on='Terminal.ConductingEquipment', + suffixes=('_ConnectivityNode', '')) + paired_injections = (topological_injections.groupby("Terminal.TopologicalNode") + .filter(lambda x: len(x) == 2)) + + # Set terminal status + updated_terminal_status = paired_injections[["ID_Terminal"]].copy().rename(columns={"ID_Terminal": "ID"}) + updated_terminal_status["KEY"] = "ACDCTerminal.connected" + updated_terminal_status["VALUE"] = "true" + + # Set Regulation off + updated_regulation_status = paired_injections[["ID"]].copy() + updated_regulation_status["KEY"] = "EquivalentInjection.regulationStatus" + updated_regulation_status["VALUE"] = "false" + + # Set P to 0 + updated_p_value = paired_injections[["ID"]].copy() + updated_p_value["KEY"] = "EquivalentInjection.p" + updated_p_value["VALUE"] = 0 + + # Set Q to 0 + updated_q_value = paired_injections[["ID"]].copy() + updated_q_value["KEY"] = "EquivalentInjection.q" + updated_q_value["VALUE"] = 0 + return cgm_ssh_data.update_triplet_from_triplet(pd.concat([updated_regulation_status, updated_p_value, updated_q_value], ignore_index=True), add=False) diff --git a/emf/model_merger/temporary_fixes.py b/emf/model_merger/temporary_fixes.py deleted file mode 100644 index 0b13bfda..00000000 --- a/emf/model_merger/temporary_fixes.py +++ /dev/null @@ -1,522 +0,0 @@ -import pypowsybl -import triplets -import pandas as pd -import logging - -from emf.common.integrations import elastic -from emf.common.loadflow_tool.helper import create_opdm_objects, get_model_outages, get_network_elements, get_opdm_data_from_models -from emf.model_merger.merge_functions import (load_opdm_data, create_sv_and_updated_ssh, fix_sv_shunts, - fix_sv_tapsteps, remove_duplicate_sv_voltages, - remove_small_islands, check_and_fix_dependencies, - export_to_cgmes_zip, - configure_paired_boundarypoint_injections_by_nodes, - ) - - -logger = logging.getLogger(__name__) - - -def check_switch_terminals(input_data: pd.DataFrame, column_name: str): - """ - Checks if column of a dataframe contains only one value - :param input_data: input data frame - :param column_name: name of the column to check - return True if different values are in column, false otherwise - """ - data_slice = (input_data.reset_index())[column_name] - return not pd.Series(data_slice[0] == data_slice).all() - - -def get_not_retained_switches_between_nodes(original_data): - """ - For the loadflow open all the non-retained switches that connect different topological nodes - Currently it is seen to help around 9 to 10 Kirchhoff 1st law errors from 2 TSOs - :param original_data: original models in triplets format - :return: updated original data - """ - updated_switches = False - original_models = get_opdm_data_from_models(original_data) - not_retained_switches = original_models[(original_models['KEY'] == 'Switch.retained') - & (original_models['VALUE'] == "false")][['ID']] - closed_switches = original_models[(original_models['KEY'] == 'Switch.open') - & (original_models['VALUE'] == 'false')] - not_retained_closed = not_retained_switches.merge(closed_switches[['ID']], on='ID') - terminals = original_models.type_tableview('Terminal').rename_axis('Terminal').reset_index() - terminals = terminals[['Terminal', - # 'ACDCTerminal.connected', - 'Terminal.ConductingEquipment', - 'Terminal.TopologicalNode']] - not_retained_terminals = (terminals.rename(columns={'Terminal.ConductingEquipment': 'ID'}) - .merge(not_retained_closed, on='ID')) - if not_retained_terminals.empty: - return original_data, updated_switches - between_tn = ((not_retained_terminals.groupby('ID')[['Terminal.TopologicalNode']] - .apply(lambda x: check_switch_terminals(x, 'Terminal.TopologicalNode'))) - .reset_index(name='same_TN')) - between_tn = between_tn[between_tn['same_TN']] - return between_tn - - -def open_switches_in_network(network_pre_instance: pypowsybl.network.Network, switches_dataframe: pd.DataFrame): - """ - Opens switches in loaded network given bu dataframe (uses ID for merging) - :param network_pre_instance: pypowsybl Network instance where igms are loaded in - :param switches_dataframe: dataframe - """ - logger.info(f"Opening {len(switches_dataframe.index)} switches") - switches = network_pre_instance.get_switches(all_attributes=True).reset_index() - switches = switches.merge(switches_dataframe[['ID']].rename(columns={'ID': 'id'}), on='id') - non_retained_closed = switches.merge(switches_dataframe.rename(columns={'ID': 'id'}), - on='id')[['id', 'open']] - non_retained_closed['open'] = True - network_pre_instance.update_switches(non_retained_closed.set_index('id')) - return network_pre_instance - - -def run_pre_merge_processing(input_models, merging_area): - - # TODO warning logs for temp fix functions - assembled_data = load_opdm_data(input_models) - assembled_data = triplets.cgmes_tools.update_FullModel_from_filename(assembled_data) - assembled_data = configure_paired_boundarypoint_injections_by_nodes(assembled_data) - escape_upper_xml = assembled_data[assembled_data['VALUE'].astype(str).str.contains('.XML')] - between_tn = get_not_retained_switches_between_nodes(assembled_data) - if not escape_upper_xml.empty: - escape_upper_xml['VALUE'] = escape_upper_xml['VALUE'].str.replace('.XML', '.xml') - assembled_data = triplets.rdf_parser.update_triplet_from_triplet(assembled_data, escape_upper_xml, update=True, add=False) - - input_models = create_opdm_objects([export_to_cgmes_zip([assembled_data])]) - - return input_models, between_tn - - -def check_net_interchanges(cgm_sv_data, cgm_ssh_data, original_models, fix_errors: bool = False, - threshold: float = None): - """ - An attempt to calculate the net interchange 2 values and check them against those provided in ssh profiles - :param cgm_sv_data: merged sv profile - :param cgm_ssh_data: merged ssh profile - :param original_models: original profiles - :param fix_errors: injects new calculated flows into merged ssh profiles - :param threshold: specify threshold if needed - :return (updated) ssh profiles - """ - original_models = get_opdm_data_from_models(model_data=original_models) - try: - control_areas = (original_models.type_tableview('ControlArea') - .rename_axis('ControlArea') - .reset_index())[['ControlArea', 'ControlArea.netInterchange', 'ControlArea.pTolerance', - 'IdentifiedObject.energyIdentCodeEic', 'IdentifiedObject.name']] - except KeyError: - control_areas = original_models.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() - ssh_areas = cgm_ssh_data.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() - control_areas = control_areas.merge(ssh_areas, on='ControlArea')[['ControlArea', 'ControlArea.netInterchange', - 'ControlArea.pTolerance', - 'IdentifiedObject.energyIdentCodeEic', - 'IdentifiedObject.name']] - tie_flows = (original_models.type_tableview('TieFlow') - .rename_axis('TieFlow').rename(columns={'TieFlow.ControlArea': 'ControlArea', - 'TieFlow.Terminal': 'Terminal'}) - .reset_index())[['ControlArea', 'Terminal', 'TieFlow.positiveFlowIn']] - tie_flows = tie_flows.merge(control_areas[['ControlArea']], on='ControlArea') - try: - terminals = (original_models.type_tableview('Terminal') - .rename_axis('Terminal').reset_index())[['Terminal', 'ACDCTerminal.connected']] - except KeyError: - terminals = (original_models.type_tableview('Terminal') - .rename_axis('Terminal').reset_index())[['Terminal']] - tie_flows = tie_flows.merge(terminals, on='Terminal') - try: - power_flows_pre = (original_models.type_tableview('SvPowerFlow') - .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) - .reset_index())[['Terminal', 'SvPowerFlow.p']] - tie_flows = tie_flows.merge(power_flows_pre, on='Terminal', how='left') - except Exception: - logger.error(f"Was not able to get tie flows from original models") - power_flows_post = (cgm_sv_data.type_tableview('SvPowerFlow') - .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) - .reset_index())[['Terminal', 'SvPowerFlow.p']] - - tie_flows = tie_flows.merge(power_flows_post, on='Terminal', how='left', - suffixes=('_pre', '_post')) - try: - tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p_pre', 'SvPowerFlow.p_post']] - .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) - .rename_axis('ControlArea').reset_index()) - except KeyError: - tie_flows_grouped = ((tie_flows.groupby('ControlArea')[['SvPowerFlow.p']] - .agg(lambda x: pd.to_numeric(x, errors='coerce').sum())) - .rename_axis('ControlArea').reset_index()) - tie_flows_grouped = tie_flows_grouped.rename(columns={'SvPowerFlow.p': 'SvPowerFlow.p_post'}) - tie_flows_grouped = control_areas.merge(tie_flows_grouped, on='ControlArea') - if threshold and threshold > 0: - tie_flows_grouped['Exceeded'] = (abs(tie_flows_grouped['ControlArea.netInterchange'] - - tie_flows_grouped['SvPowerFlow.p_post']) > threshold) - else: - tie_flows_grouped['Exceeded'] = (abs(tie_flows_grouped['ControlArea.netInterchange'] - - tie_flows_grouped['SvPowerFlow.p_post']) > - tie_flows_grouped['ControlArea.pTolerance']) - net_interchange_errors = tie_flows_grouped[tie_flows_grouped.eval('Exceeded')] - if not net_interchange_errors.empty: - if threshold > 0: - logger.warning(f"Found {len(net_interchange_errors.index)} possible net interchange_2 problems " - f"over {threshold}") - else: - logger.warning(f"Found {len(net_interchange_errors.index)} possible net interchange_2 problems") - # print(net_interchange_errors.to_string()) - if fix_errors: - logger.warning(f"Updating {len(net_interchange_errors.index)} interchanges to new values") - new_areas = cgm_ssh_data.type_tableview('ControlArea').reset_index()[['ID', - 'ControlArea.pTolerance', 'Type']] - new_areas = new_areas.merge(net_interchange_errors[['ControlArea', 'SvPowerFlow.p_post']] - .rename(columns={'ControlArea': 'ID', - 'SvPowerFlow.p_post': 'ControlArea.netInterchange'}), on='ID') - cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(cgm_ssh_data, new_areas) - return cgm_ssh_data - - -def check_non_boundary_equivalent_injections(cgm_sv_data, - cgm_ssh_data, - original_models, - threshold: float = 0, - fix_errors: bool = False): - """ - Checks equivalent injections that are not on boundary topological nodes - :param cgm_sv_data: merged SV profile - :param cgm_ssh_data: merged SSH profile - :param original_models: igms - :param threshold: threshold for checking - :param fix_errors: if true then copies values from sv profile to ssh profile - :return cgm_ssh_data - """ - original_models = get_opdm_data_from_models(model_data=original_models) - boundary_nodes = original_models.query('KEY == "TopologicalNode.boundaryPoint" & VALUE == "true"')[['ID']] - terminals = (original_models.type_tableview('Terminal').rename_axis('SvPowerFlow.Terminal').reset_index() - .merge(boundary_nodes.rename(columns={'ID': 'Terminal.TopologicalNode'}), - on='Terminal.TopologicalNode', how='outer', indicator=True))[['SvPowerFlow.Terminal', - 'Terminal.ConductingEquipment', - '_merge']] - terminals = terminals[terminals['_merge'] == 'left_only'][['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] - return check_all_kind_of_injections(cgm_sv_data=cgm_sv_data, - cgm_ssh_data=cgm_ssh_data, - original_models=original_models, - injection_name='EquivalentInjection', - fields_to_check={'SvPowerFlow.p': 'EquivalentInjection.p'}, - threshold=threshold, - terminals=terminals, - fix_errors=fix_errors) - - -def check_all_kind_of_injections(cgm_sv_data, - cgm_ssh_data, - original_models, - injection_name: str = 'ExternalNetworkInjection', - fields_to_check: dict = None, - fix_errors: bool = False, - threshold: float = 0, - terminals: pd.DataFrame = None, - report_sum: bool = True): - """ - Compares the given cgm ssh injection values to the corresponding sv powerflow values in cgm sv - :param cgm_sv_data: merged SV profile - :param cgm_ssh_data: merged SSH profile - :param original_models: igms - :param injection_name: name of the injection - :param fields_to_check: dictionary where key is the field in powerflow and value is the field in injection - :param fix_errors: if true then copies values from sv profile to ssh profile - :param threshold: max allowed mismatch - :param terminals: optional, can give dataframe of terminals as input - :param report_sum: if true prints sum of injections and powerflows to console - :return cgm_ssh_data - """ - if not fields_to_check: - return cgm_ssh_data - fixed_fields = ['ID'] - original_models = get_opdm_data_from_models(model_data=original_models) - try: - original_injections = original_models.type_tableview(injection_name).reset_index() - injections = cgm_ssh_data.type_tableview(injection_name).reset_index() - except AttributeError: - logger.info(f"SSH profile doesn't contain data about {injection_name}") - return cgm_ssh_data - injections_reduced = injections[[*fixed_fields, *fields_to_check.values()]] - original_injections_reduced = original_injections[[*fixed_fields, *fields_to_check.values()]] - injections_reduced = injections_reduced.merge(original_injections_reduced, on='ID', suffixes=('', '_org')) - if terminals is None: - terminals = (original_models.type_tableview('Terminal') - .rename_axis('SvPowerFlow.Terminal') - .reset_index())[['SvPowerFlow.Terminal', 'Terminal.ConductingEquipment']] - flows = (cgm_sv_data.type_tableview('SvPowerFlow') - .reset_index())[[*['SvPowerFlow.Terminal'], *fields_to_check.keys()]] - terminals = terminals.merge(flows, on='SvPowerFlow.Terminal') - terminals = terminals.merge(injections_reduced, left_on='Terminal.ConductingEquipment', right_on='ID') - - filtered_list = [] - for flow_field, injection_field in fields_to_check.items(): - filtered_list.append(terminals[abs(terminals[injection_field] - terminals[flow_field]) > threshold]) - if report_sum: - logger.info(f"IGM {injection_field} = {terminals[injection_field + '_org'].sum()} vs " - f"CGM {injection_field} = {terminals[injection_field].sum()} vs " - f"CGM {flow_field} = {terminals[flow_field].sum()}") - if not filtered_list: - return cgm_ssh_data - filtered = pd.concat(filtered_list).drop_duplicates().reset_index(drop=True) - if not filtered.empty: - logger.warning(f"Found {len(filtered.index)} mismatches between {injection_name} and flow values on terminals") - print(filtered.to_string()) - if fix_errors: - logger.info(f"Updating {injection_name} values from terminal flow values") - injections_update = injections.merge(filtered[[*fixed_fields, *fields_to_check.keys()]]) - injections_update = injections_update.drop(columns=fields_to_check.values()) - injections_update = injections_update.rename(columns=fields_to_check) - cgm_ssh_data = triplets.rdf_parser.update_triplet_from_tableview(data=cgm_ssh_data, - tableview=injections_update, - update=True, - add=False) - return cgm_ssh_data - - -def run_post_merge_processing(input_models: list, - merged_model: object, - task_properties: dict, - small_island_size: str, - enable_temp_fixes: bool, - time_horizon: str | None = None): - - time_horizon = time_horizon or task_properties["time_horizon"] - scenario_datetime = task_properties["timestamp_utc"] - merging_area = task_properties["merge_type"] - merging_entity = task_properties["merging_entity"] - mas = task_properties["mas"] - version = task_properties["version"] - - models_as_triplets = load_opdm_data(input_models) - sv_data, ssh_data = create_sv_and_updated_ssh(merged_model, input_models, models_as_triplets, - scenario_datetime, time_horizon, - version, merging_area, - merging_entity, mas) - fix_net_interchange_errors = task_properties.get('fix_net_interchange2', False) - net_interchange_threshold = int(task_properties.get('net_interchange2_threshold', 200)) - injection_threshold = 0.1 - fix_injection_errors = True - if enable_temp_fixes: - sv_data = fix_sv_shunts(sv_data, models_as_triplets) - sv_data = fix_sv_tapsteps(sv_data, ssh_data) - sv_data = remove_small_islands(sv_data, int(small_island_size)) - sv_data = remove_duplicate_sv_voltages(cgm_sv_data=sv_data, original_data=models_as_triplets) - sv_data = check_and_fix_dependencies(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=models_as_triplets) - #sv_data, ssh_data = disconnect_equipment_if_flow_sum_not_zero(cgm_sv_data=sv_data, cgm_ssh_data=ssh_data, original_data=models_as_triplets) fix implemented in pypowsybl 1.8.1 - ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, - cgm_sv_data=sv_data, - original_models=models_as_triplets, - injection_name='EnergySource', - threshold=injection_threshold, - fields_to_check={'SvPowerFlow.p': 'EnergySource.activePower'}, - fix_errors=fix_injection_errors) - ssh_data = check_all_kind_of_injections(cgm_ssh_data=ssh_data, - cgm_sv_data=sv_data, - original_models=models_as_triplets, - injection_name='ExternalNetworkInjection', - fields_to_check={'SvPowerFlow.p': 'ExternalNetworkInjection.p'}, - threshold=injection_threshold, - fix_errors=fix_injection_errors) - ssh_data = check_non_boundary_equivalent_injections(cgm_sv_data=sv_data, - cgm_ssh_data=ssh_data, - original_models=models_as_triplets, - threshold=injection_threshold, - fix_errors=fix_injection_errors) - try: - ssh_data = check_net_interchanges(cgm_sv_data=sv_data, - cgm_ssh_data=ssh_data, - original_models=models_as_triplets, - fix_errors=fix_net_interchange_errors, - threshold=net_interchange_threshold) - except KeyError: - logger.warning(f"No fields for netInterchange") - - return sv_data, ssh_data - - -def fix_model_outages(merged_model: object, tso_list: list, scenario_datetime: str, time_horizon: str, debug: bool = False): - - area_map = {"LITGRID": "Lithuania", "AST": "Latvia", "ELERING": "Estonia"} - outage_areas = [area_map.get(item, item) for item in tso_list] - - elk_service = elastic.Elastic() - - # Get outage eic-mrid mapping - mrid_map = elk_service.get_docs_by_query(index='config-network*', query={"match_all": {}}, size=10000) - mrid_map['mrid'] = mrid_map['mrid'].str.lstrip('_') - - # Get latest UAP parse date - if time_horizon == 'MO': - merge_type = "Month" - elif time_horizon == 'YR': - merge_type = "Year" - else: - merge_type = "Week" - - body = {"size": 1, "query": {"bool": {"must": [{"match": {"Merge": merge_type}}]}}, - "sort": [{"reportParsedDate": {"order": "desc"}}], "fields": ["reportParsedDate"]} - last_uap_version = elk_service.client.search(index='opc-outages-baltics*', body=body)['hits']['hits'][0]['fields']['reportParsedDate'][0] - - # Query for latest outage UAP - uap_query = {"bool": {"must": [{"match": {"reportParsedDate": f"{last_uap_version}"}}, - {"match": {"Merge": merge_type}}]}} - uap_outages = elk_service.get_docs_by_query(index='opc-outages-baltics*', query=uap_query, size=10000) - uap_outages = uap_outages.merge(mrid_map[['eic', 'mrid']], how='left', on='eic', indicator=True).rename(columns={"mrid": 'grid_id'}) - unmapped_outages = uap_outages[uap_outages['_merge'] == 'left_only'] - - if not unmapped_outages.empty: - logger.error(f"Unable to map these outage mRIDs: {unmapped_outages['name'].values}") - - # Filter outages according to model scenario date and replaced area - filtered_outages = uap_outages[(uap_outages['start_date'] <= scenario_datetime) & (uap_outages['end_date'] >= scenario_datetime)] - filtered_outages = filtered_outages[filtered_outages['Area'].isin(outage_areas)] - mapped_outages = filtered_outages[~filtered_outages['grid_id'].isna()] - - # Get disconnected elements in network model - model_outages = pd.DataFrame(get_model_outages(network=merged_model.network)) - mapped_model_outages = pd.merge(model_outages, mrid_map, left_on='grid_id', right_on='mrid', how='inner') - model_area_map = {"LITGRID": "LT", "AST": "LV", "ELERING": "EE"} - model_outage_areas = [model_area_map.get(item, item) for item in tso_list] - filtered_model_outages = mapped_model_outages[mapped_model_outages['country'].isin(model_outage_areas)] - - # Include cross-border lines for reconnection (both dangling lines) - dangling_lines = get_network_elements(network=merged_model.network, - element_type=pypowsybl.network.ElementType.DANGLING_LINE).reset_index(names=['grid_id']) - border_lines = dangling_lines[dangling_lines['pairing_key'].isin(model_outages['pairing_key'])] - relevant_border_lines = border_lines[border_lines['country'].isin(['LT', 'LV', 'EE'])] - additional_dangling_lines = dangling_lines[dangling_lines['pairing_key'].isin(relevant_border_lines['pairing_key'])] - - # Merged dataframe of network elements to be reconnected - filtered_model_outages = pd.concat([filtered_model_outages, additional_dangling_lines]).drop_duplicates(subset='grid_id') - filtered_model_outages = filtered_model_outages.where(pd.notnull(filtered_model_outages), None) - - # rename columns - filtered_model_outages = filtered_model_outages.copy()[['name', 'grid_id', 'eic']].rename(columns={'grid_id': 'mrid'}) - mapped_outages = mapped_outages.copy()[['name', 'grid_id', 'eic']].rename(columns={'grid_id': 'mrid'}) - - logger.info("Fixing outages inside merged model") - - # Reconnecting outages from network-config list - outages_updated = {} - for index, outage in filtered_model_outages.iterrows(): - try: - if merged_model.network.connect(outage['mrid']): - logger.info(f"Successfully reconnected: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages = True - outage_dict = outage.to_dict() - outage_dict.update({'status': 'connected'}) - outages_updated[outage_dict['mrid']] = outage_dict - else: - if uap_outages['grid_id'].str.contains(outage['mrid']).any(): - logger.info(f"Element is already connected: {outage['name']} [mrid: {outage['mrid']}]") - else: - logger.error(f"Failed to connect element: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - except Exception as e: - logger.error((e, outage['name'])) - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - merged_model.outages = False - continue - - # Applying outages from UAP - for index, outage in mapped_outages.iterrows(): - try: - if merged_model.network.disconnect(outage['mrid']): - logger.info(f"Successfully disconnected: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages = True - outage_dict = outage.to_dict() - outage_dict.update({'status': 'disconnected'}) - outages_updated[outage_dict['mrid']] = outage_dict - else: - if uap_outages['grid_id'].str.contains(outage['mrid']).any(): - logger.info(f"Element is already in outage: {outage['name']} [mrid: {outage['mrid']}]") - else: - logger.error(f"Failed to disconnect element: {outage['name']} [mrid: {outage['mrid']}]") - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - except Exception as e: - logger.error((e, outage['name'])) - merged_model.outages_unmapped.extend([{"name": outage['name'], "mrid": outage['mrid'], "eic": outage['eic']}]) - merged_model.outages = False - continue - - # Keep only important keys of updated outages - merged_model.outages_updated = list(outages_updated.values()) - - if merged_model.outages_unmapped: - merged_model.outages = False - - return merged_model - - -def fix_igm_ssh_vs_cgm_ssh_error(network_pre_instance: pypowsybl.network.Network): - """ - Implements various fixes to suppress igm ssh vs cgm ssh error - 1) Get all generators and remove them from slack distribution - 2) If generators have target_p outside the endpoints ('limits') of a curve then set it to be within - 3) Condensers p should not be modified so if it is not 0 then it sets the target_p to equal the existing p - :param network_pre_instance: pypowsybl Network instance where igms are loaded in - :return updated network_pre_instance - """ - try: - all_generators = network_pre_instance.get_elements(element_type=pypowsybl.network.ElementType.GENERATOR, - all_attributes=True).reset_index() - generators_mask = (all_generators['CGMES.synchronousMachineOperatingMode'].str.contains('generator')) - not_generators = all_generators[~generators_mask] - generators = all_generators[generators_mask] - curve_points = (network_pre_instance - .get_elements(element_type=pypowsybl.network.ElementType.REACTIVE_CAPABILITY_CURVE_POINT, - all_attributes=True).reset_index()) - curve_limits = (curve_points.merge(generators[['id']], on='id') - .groupby('id').agg(curve_p_min=('p', 'min'), curve_p_max=('p', 'max'))).reset_index() - curve_generators = generators.merge(curve_limits, on='id') - # low end can be zero - curve_generators = curve_generators[(curve_generators['target_p'] > curve_generators['curve_p_max']) | - ((curve_generators['target_p'] > 0) & - (curve_generators['target_p'] < curve_generators['curve_p_min']))] - if not curve_generators.empty: - logger.warning(f"Found {len(curve_generators.index)} generators for " - f"which p > max(reactive capacity curve(p)) or p < min(reactive capacity curve(p))") - - # Solution 1: set max_p from curve max, it should contain p on target-p - upper_limit_violated = curve_generators[(curve_generators['max_p'] > curve_generators['curve_p_max'])] - if not upper_limit_violated.empty: - logger.warning(f"Updating max p from curve for {len(upper_limit_violated.index)} generators") - upper_limit_violated['max_p'] = upper_limit_violated['curve_p_max'] - network_pre_instance.update_generators(upper_limit_violated[['id', 'max_p']].set_index('id')) - - lower_limit_violated = curve_generators[(curve_generators['min_p'] < curve_generators['curve_p_min'])] - if not lower_limit_violated.empty: - logger.warning(f"Updating min p from curve for {len(lower_limit_violated.index)} generators") - lower_limit_violated['min_p'] = lower_limit_violated['curve_p_min'] - network_pre_instance.update_generators(lower_limit_violated[['id', 'min_p']].set_index('id')) - - # Solution 2: discard generator from participating - extensions = network_pre_instance.get_extensions('activePowerControl') - remove_curve_generators = extensions.merge(curve_generators[['id']], - left_index=True, right_on='id') - if not remove_curve_generators.empty: - remove_curve_generators['participate'] = False - network_pre_instance.update_extensions('activePowerControl', - remove_curve_generators.set_index('id')) - condensers = all_generators[(all_generators['CGMES.synchronousMachineType'].str.contains('condenser')) - & (abs(all_generators['p']) > 0) - & (abs(all_generators['target_p']) == 0)] - # Fix condensers that have p not zero by setting their target_p to equal to p - if not condensers.empty: - logger.warning(f"Found {len(condensers.index)} condensers for which p ~= 0 & target_p = 0") - condensers.loc[:, 'target_p'] = condensers['p'] * (-1) - network_pre_instance.update_generators(condensers[['id', 'target_p']].set_index('id')) - # Remove all not generators from active power distribution - if not not_generators.empty: - logger.warning(f"Removing {len(not_generators.index)} machines from power distribution") - extensions = network_pre_instance.get_extensions('activePowerControl') - remove_not_generators = extensions.merge(not_generators[['id']], left_index=True, right_on='id') - remove_not_generators['participate'] = False - remove_not_generators = remove_not_generators.set_index('id') - network_pre_instance.update_extensions('activePowerControl', remove_not_generators) - except Exception as ex: - logger.warning(f"Unable to pre-process for igm-cgm-ssh error: {ex}") - return network_pre_instance diff --git a/emf/model_merger/worker.py b/emf/model_merger/worker.py index f052cae0..f5dd2a51 100644 --- a/emf/model_merger/worker.py +++ b/emf/model_merger/worker.py @@ -2,6 +2,10 @@ from uuid import uuid4 from emf.common.logging import custom_logger +# Supress FutureWarnings from triplets library cause by pandas to_numeric errors ignore deprecation +import warnings +warnings.filterwarnings('ignore', category=FutureWarning) + # Initialize custom logger logger = logging.getLogger(__name__) elk_handler = custom_logger.initialize_custom_logger(extra={'worker': 'model-merger', 'worker_uuid': str(uuid4())}) @@ -17,7 +21,10 @@ parse_app_properties(caller_globals=globals(), path=config.paths.cgm_worker.merger) # RabbitMQ consumer implementation -consumer = rabbit.RMQConsumer(queue=INPUT_RABBIT_QUE, message_handlers=[HandlerMergeModels()]) +consumer = rabbit.RMQConsumer(queue=INPUT_RABBIT_QUE, + message_handlers=[HandlerMergeModels()], + forward=OUTPUT_RMQ_EXCHANGE, + ) try: consumer.run() diff --git a/emf/model_quality/__init__.py b/emf/model_quality/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/emf/model_quality/model_quality.py b/emf/model_quality/model_quality.py new file mode 100644 index 00000000..18fefb76 --- /dev/null +++ b/emf/model_quality/model_quality.py @@ -0,0 +1,85 @@ +import logging +import pandas as pd +import config +import json +from emf.common.config_parser import parse_app_properties +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets +from emf.common.integrations import elastic, minio_api +from emf.common.integrations.object_storage import models +from triplets.rdf_parser import load_all_to_dataframe +from emf.model_quality.model_statistics import get_system_metrics +from emf.model_quality.quality_functions import generate_quality_report, process_zipped_cgm + +logger = logging.getLogger(__name__) + +parse_app_properties(caller_globals=globals(), path=config.paths.model_quality.model_quality) + + +class HandlerModelQuality: + + def __init__(self): + self.minio_service = minio_api.ObjectStorage() + self.elastic_service = elastic.Elastic() + + def handle(self, message: bytes, properties: dict, **kwargs): + + # Load OPDM metadata objects from binary to json + model_metadata = json.loads(message) + object_type = properties.headers['opde:Object-Type'] + + if object_type == 'CGM': + model_data = self.minio_service.download_object(model_metadata.get('minio-bucket', 'opde-confidential-models'), + model_metadata.get('pmd:content-reference')) + logger.info(f"Loading merged model: {model_metadata['name']}") + unzipped = process_zipped_cgm(model_data) + network= load_all_to_dataframe(unzipped) + + elif object_type == 'IGM': + latest_boundary = models.get_latest_boundary() + model_data = [models.get_content(metadata=opdm_object) for opdm_object in model_metadata] + try: + for opdm_object in model_data: + network = load_opdm_objects_to_triplets(opdm_objects=[opdm_object, latest_boundary]) + except: + logger.error("Failed to load IGM") + network = pd.DataFrame + else: + logger.error("Incorrect or missing metadata") + network = pd.DataFrame + + if not network.empty: + qa_report = generate_quality_report(network, object_type, model_metadata) + try: + # TODO move statistics function to quality functions file or move statistics file to quality directory + model_statistics = get_system_metrics(network) + except Exception as e: + model_statistics = {} + logger.error(f"Failed to get model statistics: {e}") + else: + raise TypeError("Model was not loaded correctly, either missing in MinIO or incorrect data") + + # TODO align naming for opdm_objects + if model_statistics: + model_statistics.update({k: v for k, v in opdm_object.items() if k.startswith('@')}) + model_statistics.update(properties.headers) + try: + response = self.elastic_service.send_to_elastic(index=ELK_STATISTICS_INDEX, json_message=model_statistics) + except Exception as error: + logger.error(f"Statistics report sending to Elastic failed: {error}") + + logger.info(f"Statistics report sent to elastic index: '{ELK_STATISTICS_INDEX}'") + else: + raise TypeError("Statistics report generator failed, data not sent") + + # Send validation report to Elastic + if qa_report: + try: + response = self.elastic_service.send_to_elastic(index=ELK_QUALITY_INDEX, json_message=qa_report) + except Exception as error: + logger.error(f"Validation report sending to Elastic failed: {error}") + + logger.info(f"Quality report sent to elastic index: '{ELK_QUALITY_INDEX}'") + else: + logger.error("Error, quality report generator failed, data not sent") + + return message, properties diff --git a/emf/model_validator/model_statistics.py b/emf/model_quality/model_statistics.py similarity index 85% rename from emf/model_validator/model_statistics.py rename to emf/model_quality/model_statistics.py index 9df04d99..952a431e 100644 --- a/emf/model_validator/model_statistics.py +++ b/emf/model_quality/model_statistics.py @@ -87,8 +87,23 @@ def parse_type_and_attribute(type_and_attr_string, default_attr="ID"): def get_tieflow_data(data): logger.info("Getting Tieflow Data") - tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal->ConnectivityNode") - tieflow_data["BoundaryPoint.isDirectCurrent"] = tieflow_data["IdentifiedObject.description"].str.startswith("HVDC") + try: + tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal->ConnectivityNode") + except: + try: + tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal-ConnectivityNode") + except: + tieflow_data = type_tableview_merge(data, "ControlArea<-TieFlow->Terminal-ConnectivityNode") + + + try: + tieflow_data["BoundaryPoint.isDirectCurrent"] = tieflow_data["IdentifiedObject.description"].str.startswith("HVDC") + except: + try: + tieflow_data["BoundaryPoint.isDirectCurrent"] = tieflow_data["IdentifiedObject.description_ConnectivityNode"].str.startswith("HVDC") + except: + tieflow_data["BoundaryPoint.isDirectCurrent"] = False + # TODO - for CGMES3/CIM17 get also the Boundary objects and use correct field to identify HVDC # Add Injections @@ -169,7 +184,7 @@ def get_system_metrics(data, tieflow_data=None, load_and_generation=None): losses_coefficient = losses / (abs(load) + abs(generation) + tieflow_abs_ei_p) if tieflow_abs_ei_p else None # Returning the computed metrics as a dictionary - return { + result = { 'total_load': load, 'generation': generation, 'losses': losses, @@ -178,9 +193,28 @@ def get_system_metrics(data, tieflow_data=None, load_and_generation=None): 'tieflow_np': tieflow_np, 'tieflow_acnp': tieflow_acnp, 'tieflow_hvdc': tieflow_hvdc, + } + # Fixes for ELK data storage + result = { + outer_key: { + key.replace('.p', '_p').replace('.q', '_q'): value + for key, value in inner_dict.items() + } if isinstance(inner_dict, dict) else inner_dict + for outer_key, inner_dict in result.items() } + tieflow_hvdc = [{**inner_key, 'eic': outer_key} for outer_key, inner_key in tieflow_hvdc.items()] + tieflow_hvdc = [ + {key.replace('.p', '_p').replace('.q', '_q'): value + for key, value in item.items() + } if isinstance(item, dict) else item + for item in tieflow_hvdc + ] + result['tieflow_hvdc'] = tieflow_hvdc + + return result + if __name__ == "__main__": import triplets import pandas diff --git a/emf/model_quality/quality_functions.py b/emf/model_quality/quality_functions.py new file mode 100644 index 00000000..2bb20e38 --- /dev/null +++ b/emf/model_quality/quality_functions.py @@ -0,0 +1,174 @@ +from io import BytesIO +from zipfile import ZipFile +import logging + +logger = logging.getLogger(__name__) + +def generate_quality_report(network, object_type, model_metadata): + + report = {} + + if object_type == "CGM" and model_metadata['pmd:Area'] == 'BA': + + # Check Kruonis generators + generators = network.type_tableview('SynchronousMachine').rename_axis('Terminal').reset_index() + kruonis_generators = generators[generators['IdentifiedObject.name'].str.contains('KHAE_G')] + + if not kruonis_generators.empty: + gen_count = kruonis_generators[kruonis_generators['RotatingMachine.p'] > 0].shape[0] + flag = gen_count < 3 + report.update({"kruonis_generators": gen_count, "kruonis_check": flag}) + else: + report.update({"kruonis_generators": None, "kruonis_check": False}) + + + # Check LT-PL crossborder flow + try: + control_areas = (network.type_tableview('ControlArea') + .rename_axis('ControlArea') + .reset_index())[['ControlArea', 'ControlArea.netInterchange', 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', 'IdentifiedObject.name']] + except KeyError: + control_areas = network.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + ssh_areas = network.type_tableview('ControlArea').rename_axis('ControlArea').reset_index() + control_areas = control_areas.merge(ssh_areas, on='ControlArea')[ + ['ControlArea', 'ControlArea.netInterchange', + 'ControlArea.pTolerance', + 'IdentifiedObject.energyIdentCodeEic', + 'IdentifiedObject.name']] + tie_flows = (network.type_tableview('TieFlow') + .rename_axis('TieFlow').rename(columns={'TieFlow.ControlArea': 'ControlArea', + 'TieFlow.Terminal': 'Terminal'}) + .reset_index())[['ControlArea', 'Terminal', 'TieFlow.positiveFlowIn']] + tie_flows = tie_flows.merge(control_areas[['ControlArea']], on='ControlArea') + try: + terminals = (network.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal', 'ACDCTerminal.connected']] + except KeyError: + terminals = (network.type_tableview('Terminal') + .rename_axis('Terminal').reset_index())[['Terminal']] + tie_flows = tie_flows.merge(terminals, on='Terminal') + try: + power_flows_pre = (network.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + tie_flows = tie_flows.merge(power_flows_pre, on='Terminal', how='left') + except Exception: + logger.error(f"Was not able to get tie flows from original models") + power_flows_post = (network.type_tableview('SvPowerFlow') + .rename(columns={'SvPowerFlow.Terminal': 'Terminal'}) + .reset_index())[['Terminal', 'SvPowerFlow.p']] + + tie_flows = tie_flows.merge(power_flows_post, on='Terminal', how='left', + suffixes=('_pre', '_post')) + + # TODO double check correct limit value + # BORDER_LIMIT = 250 + # d_lines = network.get_dangling_lines(all_attributes=True) + # LT_PL_lines = d_lines[d_lines['name'].str.contains('Alytus-Elk')] + # if LT_PL_lines: + # flow_sum = LT_PL_lines['p'].sum() + # flag = flow_sum < BORDER_LIMIT + # report.update({"lt_pl_flow": flow_sum, "lt_pl_xborder_check": flag}) + # else: + + # TODO fix border flow + report.update({"lt_pl_flow": None, "lt_pl_xborder_check": False}) + + # TODO remake into Triplets + # Check cross-border line inconsistencies + # TODO log all line info + # pairing_keys = d_lines.groupby('pairing_key')['connected'].nunique() + # mismatch = len(pairing_keys[pairing_keys > 1].index.tolist()) + # flag = mismatch < 1 + # report.update({"xb_mismatch": mismatch, "xb_consitency_check": flag}) + + # TODO Check model outage mismatch with outage plan + # model_outages = pd.DataFrame(get_model_outages(network=network)) + + # TODO define IGM quality rules + elif object_type == "IGM": + report = model_metadata[0] + model_metadata[0].pop('opde:Component') + try: + model_metadata[0].pop('opde:Dependencies') + except: + model_metadata[0].pop('opde:DependsOn') + + report.update({"quality": "No Status"}) + + return report + + +# TODO temp function, later use common one +def query_elk_uap(index, time_horizon=None): + + from datetime import datetime + + logger.info(f"Retrieving outages from ELK index: '{index}'") + + # now represents the time of the run, in P0W case it should be current time + now = datetime.now() + now = now.strftime("%Y-%m-%dT%H:%M") + "Z" + + if time_horizon == 'WK': + merge_type_list = ['week'] + elif time_horizon == 'MO': + merge_type_list = ['week', 'month'] + elif time_horizon == 'YR': + merge_type_list = ['year'] + + query = { + "bool": { + "must": [ + {"exists": {"field": "name"}}, + {"terms": {"Merge": merge_type_list}}, + ], + "filter": [{"range": {"reportParsedDate": {"lte": now, "gte": "now-2w"}}}], + } + } + response = get_docs_by_query(index=index, query=query, size=10000, return_df=True) + result = pd.DataFrame() + + if not response.empty: + + # Get only latest report data + response['reportParsedDate'] = pd.to_datetime(response['reportParsedDate']) + response = response[response['reportParsedDate'] == response['reportParsedDate'].max()] + # Only keep latest outages + duplicated_outages = response[response.duplicated('eic', keep=False)] + latest_duplicate = duplicated_outages.groupby('eic')['date_of_last_change'].idxmax() + response = response.loc[response.index.isin(latest_duplicate) | ~response['eic'].duplicated(keep=False)] + + response = response[response['outage_type'].isin(['OUT'])] + + response = response.sort_values(by=['eic', 'start_date', 'end_date']).reset_index(drop=True) + last_end_time = {} + + # Remove outage duplicate if there is time overlap + for _, row in response.iterrows(): + eic = row['eic'] + start_time = row['start_date'] + end_time = row['end_date'] + + if eic not in last_end_time or start_time > last_end_time[eic]: + result = pd.concat([result, pd.DataFrame([row])], ignore_index=True) + last_end_time[eic] = end_time + + return result + + +def process_zipped_cgm(zipped_bytes, processed=[]): + + with ZipFile(BytesIO(zipped_bytes)) as zf: + for name in zf.namelist(): + with zf.open(name) as file: + content = file.read() + if name.endswith('.zip'): + process_zipped_cgm(content) + elif name.endswith('.xml'): + file_object = BytesIO(content) + file_object.name = name + processed.append(file_object) + + return processed diff --git a/emf/model_quality/worker.py b/emf/model_quality/worker.py new file mode 100644 index 00000000..9bffe5d1 --- /dev/null +++ b/emf/model_quality/worker.py @@ -0,0 +1,20 @@ +import config +import logging +from emf.common.integrations import rabbit +from emf.common.config_parser import parse_app_properties +from emf.model_quality.model_quality import HandlerModelQuality + +# Initialize custom logger +logger = logging.getLogger(__name__) + +parse_app_properties(caller_globals=globals(), path=config.paths.model_quality.model_quality) + +# RabbitMQ consumer implementation +consumer = rabbit.RMQConsumer(queue=INPUT_RMQ_QUEUE, + message_handlers=[HandlerModelQuality()], + ) + +try: + consumer.run() +except KeyboardInterrupt: + consumer.stop() diff --git a/emf/model_retriever/model_retriever.py b/emf/model_retriever/model_retriever.py index 82ddbafb..71be14b9 100644 --- a/emf/model_retriever/model_retriever.py +++ b/emf/model_retriever/model_retriever.py @@ -3,8 +3,8 @@ from io import BytesIO import json import triplets - -from emf.common.loadflow_tool.helper import zip_xml, create_opdm_objects +from emf.common.helpers.opdm_objects import create_opdm_objects +from emf.common.helpers.utils import zip_xml from emf.common.config_parser import parse_app_properties from emf.common.integrations import elastic, opdm, minio_api @@ -43,7 +43,7 @@ def handle(self, message: bytes, properties: dict, **kwargs): # Repackage to form of zip(xml) rdfzip_files = [] for xml in rdfxml_files: - rdfzip_files.append(zip_xml(xml)) + rdfzip_files.append(zip_xml(xml_file_object=xml)) # Create OPDM objects opdm_objects = create_opdm_objects(models=[rdfzip_files], metadata={"data-source": "PDN"}) diff --git a/emf/model_validator/model_validator.py b/emf/model_validator/model_validator.py index 4d6eafad..bcafd83b 100644 --- a/emf/model_validator/model_validator.py +++ b/emf/model_validator/model_validator.py @@ -6,13 +6,17 @@ import math import pypowsybl as pp import uuid -from emf.common.loadflow_tool.helper import attr_to_dict, load_model +import triplets from emf.common.config_parser import parse_app_properties from emf.common.integrations import elastic, minio_api from emf.common.integrations.object_storage import models -from emf.common.loadflow_tool.helper import load_opdm_data, clean_data_from_opdm_objects from emf.common.loadflow_tool import loadflow_settings +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets, clean_data_from_opdm_objects +from emf.common.helpers.loadflow import load_network_model +from emf.common.helpers.utils import attr_to_dict +from emf.common.helpers.cgmes import export_to_cgmes_zip from emf.model_validator import validator_functions +from emf.common.decorators import performance_counter logger = logging.getLogger(__name__) @@ -26,15 +30,14 @@ def __init__(self, network: pd.DataFrame): self.report = {'pre_validations': {}} def validate_non_retained_switches(self): - # TODO Use carefully as this might change switch statuses in the original model - # TODO Currently disabled returning modified data non_retained_switched_valid = True - non_retained_switches = validator_functions.check_not_retained_switches_between_nodes(self.network)[1] - if non_retained_switches: + violated_switches = validator_functions.check_not_retained_switches_between_nodes(original_data=self.network)[1] + if violated_switches: non_retained_switched_valid = False self.report['pre_validations']['non_retained_switches'] = non_retained_switched_valid def validate_kirchhoff_first_law(self): + # TODO - currently this is not used in pre-validation, but it might be useful to have it here violated_nodes = validator_functions.get_nodes_against_kirchhoff_first_law(original_models=self.network) kirchhoff_first_law_valid = True if violated_nodes.empty else False self.report['pre_validations']['kirchhoff_first_law'] = kirchhoff_first_law_valid @@ -42,8 +45,6 @@ def validate_kirchhoff_first_law(self): def run_validation(self): if json.loads(CHECK_NON_RETAINED_SWITCHES.lower()): self.validate_non_retained_switches() - if json.loads(CHECK_KIRCHHOFF_FIRST_LAW.lower()): - self.validate_kirchhoff_first_law() class PostLFValidator: @@ -56,11 +57,28 @@ def __init__(self, network: pp.network, network_triplets: pd.DataFrame): def validate_loadflow(self): """Validate load flow convergence""" - logger.info(f"Solving load flow with settings: {VALIDATION_LOAD_FLOW_SETTINGS}") - loadflow_report = pp.report.Reporter() - loadflow_result = pp.loadflow.run_ac(network=self.network, - parameters=self.loadflow_parameters, - reporter=loadflow_report) + # Set starting point of lf settings priority list + if json.loads(ENABLE_DYNAMIC_VALIDATION_SETTINGS.lower()): + settings_list = [param.strip() for param in VALIDATION_LOAD_FLOW_SETTINGS_PRIORITY.split(",")] + settings_priority = next((i for i, value in enumerate(settings_list) if value == VALIDATION_LOAD_FLOW_SETTINGS), None) + settings_list = settings_list[settings_priority:] + else: + settings_list = [VALIDATION_LOAD_FLOW_SETTINGS] + + # Run loadflow, relaxing settings after each diverging result + for lf_settings in settings_list: + loadflow_parameters = getattr(loadflow_settings, lf_settings) + logger.info(f"Solving loadflow with settings: {lf_settings}") + # loadflow_report = pp.report.Reporter() + loadflow_result = pp.loadflow.run_ac(network=self.network, + parameters=loadflow_parameters, + # reporter=loadflow_report, + ) + + if loadflow_result[0].status_text == 'Converged': + break + else: + logger.warning(f"Failed to solve loadflow with settings: {lf_settings}") # Parsing aggregated results self.report['components'] = len(loadflow_result) @@ -71,13 +89,14 @@ def validate_loadflow(self): # TODO currently storing only main island results main_component = loadflow_result[0] component_results = attr_to_dict(main_component) + logger.info(f"Loadflow status: {main_component.status.name} [settings: {lf_settings}]") component_results['status'] = component_results.get('status').value component_results['distributed_active_power'] = 0.0 if math.isnan(component_results['distributed_active_power'])\ else component_results['distributed_active_power'] self.report['loadflow'] = component_results # Validation status - self.report['loadflow_parameters'] = VALIDATION_LOAD_FLOW_SETTINGS + self.report['loadflow_parameters'] = lf_settings self.report['validations']['loadflow'] = True if main_component.status.value == 0 else False def validate_network_elements(self): @@ -98,9 +117,7 @@ def validate_network_elements(self): def validate_kirchhoff_first_law(self): """Validates possible Kirchhoff first law errors after loadflow""" - violated_nodes = pd.DataFrame() - - # Export sv profile and check it for Kirchhoff 1st law + # Export SV profile and check it for Kirchhoff 1st law export_parameters = {"iidm.export.cgmes.profiles": 'SV', "iidm.export.cgmes.naming-strategy": "cgmes-fix-all-invalid-ids"} bytes_object = self.network.save_to_binary_buffer(format="CGMES", parameters=export_parameters) @@ -124,6 +141,50 @@ def run_validation(self): self.validate_kirchhoff_first_law() +class TemporaryPreMergeModifications: + """Run pre-processing modification important for successful merge process""" + # TODO needs to keep in mind that this is meant as temporary fixes and needs to be revised often + def __init__(self, network: pd.DataFrame): + self.network = network + self.report = {'modification': {}} + + def update_header_from_file_name(self): + self.report['modification']['header_from_file_name'] = False + modified_data = triplets.cgmes_tools.update_FullModel_from_filename(self.network) + if not pd.concat([self.network, modified_data]).drop_duplicates(keep=False).empty: + self.network = modified_data + self.report['modification']['header_from_file_name'] = True + + def sanitize_file_name(self): + self.report['modification']['sanitize_file_name'] = False + xml_extension_typo = self.network[self.network['VALUE'].str.contains('.XML', regex=False, na=False)] + if not xml_extension_typo.empty: + xml_extension_typo['VALUE'] = xml_extension_typo['VALUE'].str.replace('.XML', '.xml') + self.network = triplets.rdf_parser.update_triplet_from_triplet(data=self.network, + update_data=xml_extension_typo, + update=True, + add=False) + self.report['modification']['sanitize_file_name'] = True + + def open_non_retained_switches(self): + self.report['modification']['open_non_retained_switches'] = False + # Open non-retained switches + original_data, violated_switches = validator_functions.check_not_retained_switches_between_nodes( + original_data=self.network, open_not_retained_switches=True) + if violated_switches: + self.network = original_data + self.report['modification']['open_non_retained_switches'] = True + + @performance_counter(units='seconds') + def run_pre_process_modifications(self): + self.update_header_from_file_name() + self.sanitize_file_name() + if json.loads(OPEN_NON_RETAINED_SWITCHES.lower()): + self.open_non_retained_switches() + + return self.network + + class HandlerModelsValidator: def __init__(self): @@ -141,33 +202,56 @@ def handle(self, message: bytes, properties: dict, **kwargs): start_time = time.time() # Load OPDM metadata objects from binary to json - opdm_objects = json.loads(message) + opdm_metadata = json.loads(message) # Get network models data from object storage - opdm_objects = [models.get_content(metadata=opdm_object) for opdm_object in opdm_objects] + opdm_objects = [models.get_content(metadata=opdm_object) for opdm_object in opdm_metadata] + + # Exclude BDS-type objects from validation + opdm_objects = [opdm_object for opdm_object in opdm_objects if opdm_object["opde:Object-Type"] != "BDS"] + if not opdm_objects: + logger.warning("No OPDM objects for validations or it was type of BDS, exiting validation") + return message, properties # Get the latest boundary set for validation latest_boundary = models.get_latest_boundary() - logger.info(f"Validation parameters used: {VALIDATION_LOAD_FLOW_SETTINGS}") + # logger.info(f"Validation parameters used: {VALIDATION_LOAD_FLOW_SETTINGS}") # Run network model validations for opdm_object in opdm_objects: report = {} try: # Run pre-loadflow validations - network_triplets = load_opdm_data(opdm_objects=[opdm_object, latest_boundary]) + network_triplets = load_opdm_objects_to_triplets(opdm_objects=[opdm_object, latest_boundary]) pre_lf_validation = PreLFValidator(network=network_triplets) pre_lf_validation.run_validation() # Run post-loadflow validations - network = load_model(opdm_objects=[opdm_object, latest_boundary]) + network = load_network_model(opdm_objects=[opdm_object, latest_boundary]) post_lf_validation = PostLFValidator(network=network, network_triplets=network_triplets) post_lf_validation.run_validation() + # Clean DATA from OPDM object as this is already converted to other formats + opdm_object = clean_data_from_opdm_objects(opdm_objects=[opdm_object])[0] + + # Apply pre-processing modification to models and store in Minio + pre_merge_modification = TemporaryPreMergeModifications(network=network_triplets) + network_triplets = pre_merge_modification.run_pre_process_modifications() + cgmes_modified_model = export_to_cgmes_zip([network_triplets]) + for component in opdm_object['opde:Component']: + # Map exported modified file with initial inside opdm object + cgmes_file = [i for i in cgmes_modified_model if i.name == component['opdm:Profile']['pmd:fileName']][0] + cgmes_file.name = component['opdm:Profile']['pmd:content-reference'].replace('//', '/') + logger.info(f"Uploading modified model content to Minio: {cgmes_file.name}") + self.minio_service.upload_object(file_path_or_file_object=cgmes_file, + bucket_name=opdm_object['minio-bucket'], + tags={"state": "modified"}) + # Collect both pre and post loadflow validation reports and merge report.update(pre_lf_validation.report) report.update(post_lf_validation.report) + report.update(pre_merge_modification.report) # Include relevant metadata fields report['@scenario_timestamp'] = opdm_object['pmd:scenarioDate'] @@ -176,20 +260,11 @@ def handle(self, message: bytes, properties: dict, **kwargs): report['content_reference'] = opdm_object['pmd:content-reference'] report['tso'] = opdm_object['pmd:TSO'] report['duration_s'] = round(time.time() - start_time, 3) + report['minio_bucket'] = opdm_object['minio-bucket'] except Exception as error: logger.error(f"Models validator failed with exception: {error}", exc_info=True) - - # TODO temporary monitored metrics - if opdm_object['pmd:TSO'] == "LITGRID": - try: - dl = network.get_dangling_lines() - ltpl_border_flow = dl.query("pairing_key in ['XEL_AL11', 'XEL_AL12']").p.sum() - gens = network.get_generators() - khae_gen = gens[gens['name'].str.contains('KHAE_G')].p.sum() - report["monitor"] = {"poland_border_flow": ltpl_border_flow, "khae_generation": khae_gen} - except Exception as error: - logger.error(f"Monitored metrics collection failed: {error}", exc_info=True) + continue # Define model validity valid = all(report['validations'].values()) @@ -198,7 +273,6 @@ def handle(self, message: bytes, properties: dict, **kwargs): try: logger.info("Updating OPDM metadata in Elastic with model valid status") # self.update_opdm_metadata_object(id=opdm_object['opde:Id'], body={'valid': valid}) - opdm_object = clean_data_from_opdm_objects(opdm_objects=[opdm_object])[0] opdm_object["valid"] = valid self.elastic_service.send_to_elastic_bulk( index=METADATA_ELK_INDEX, @@ -231,19 +305,19 @@ def handle(self, message: bytes, properties: dict, **kwargs): level=logging.INFO, handlers=[logging.StreamHandler(sys.stdout)] ) - #logging.getLogger('powsybl').setLevel(1) opdm = OPDM() - latest_boundary = opdm.get_latest_boundary() - available_models = opdm.get_latest_models_and_download(time_horizon='1D', - scenario_date="2025-04-28T09:30", - tso="AST") + latest_boundary = models.get_latest_boundary() + available_models = models.get_latest_models_and_download(time_horizon='1D', + scenario_date="20250706T0930", + tso="AST", + data_source='opdm') validated_models = [] # Validate models for model in available_models: - network_triplets = load_opdm_data(opdm_objects=[model, latest_boundary]) - network = load_model(opdm_objects=[model, latest_boundary]) + network_triplets = load_opdm_objects_to_triplets(opdm_objects=[model, latest_boundary]) + network = load_network_model(opdm_objects=[model, latest_boundary]) post_lf_validation = PostLFValidator(network=network, network_triplets=network_triplets) post_lf_validation.run_validation() diff --git a/emf/model_validator/validator_functions.py b/emf/model_validator/validator_functions.py index d50f4b67..c6b3f92d 100644 --- a/emf/model_validator/validator_functions.py +++ b/emf/model_validator/validator_functions.py @@ -1,7 +1,7 @@ import logging import pandas import triplets -from emf.common.loadflow_tool.helper import get_opdm_data_from_models +from emf.common.helpers.opdm_objects import load_opdm_objects_to_triplets logger = logging.getLogger(__name__) @@ -19,7 +19,7 @@ def get_nodes_against_kirchhoff_first_law(original_models, :param nodes_only: if true then return unique nodes only, if false then nodes with corresponding terminals :param sv_injection_limit: threshold for deciding whether the node is violated by sum of flows """ - original_models = get_opdm_data_from_models(model_data=original_models) + original_models = load_opdm_objects_to_triplets(opdm_objects=original_models) sv_injections = pandas.DataFrame() if cgm_sv_data is None: cgm_sv_data = original_models @@ -78,28 +78,29 @@ def check_not_retained_switches_between_nodes(original_data, open_not_retained_s :param open_not_retained_switches: if true then found switches are set to open, else it only checks and reports :return: updated original data """ - updated_switches = 0 - original_models = get_opdm_data_from_models(original_data) + violated_switches = 0 + if not isinstance(original_data, pandas.DataFrame): + original_models = load_opdm_objects_to_triplets(opdm_objects=original_data) + else: + original_models = original_data not_retained_switches = original_models[(original_models['KEY'] == 'Switch.retained') & (original_models['VALUE'] == "false")][['ID']] closed_switches = original_models[(original_models['KEY'] == 'Switch.open') & (original_models['VALUE'] == 'false')] not_retained_closed = not_retained_switches.merge(closed_switches[['ID']], on='ID') terminals = original_models.type_tableview('Terminal').rename_axis('Terminal').reset_index() - terminals = terminals[['Terminal', - # 'ACDCTerminal.connected', - 'Terminal.ConductingEquipment', - 'Terminal.TopologicalNode']] + terminals = terminals[['Terminal', 'Terminal.ConductingEquipment', 'Terminal.TopologicalNode']] not_retained_terminals = (terminals.rename(columns={'Terminal.ConductingEquipment': 'ID'}) .merge(not_retained_closed, on='ID')) if not_retained_terminals.empty: - return original_data, updated_switches + return original_data, violated_switches + between_tn = ((not_retained_terminals.groupby('ID')[['Terminal.TopologicalNode']] .apply(lambda x: check_switch_terminals(x, 'Terminal.TopologicalNode'))) .reset_index(name='same_TN')) between_tn = between_tn[between_tn['same_TN']] if not between_tn.empty: - updated_switches = len(between_tn.index) + violated_switches = len(between_tn.index) logger.warning(f"Found {len(between_tn.index)} not retained switches between topological nodes") if open_not_retained_switches: logger.warning(f"Opening not retained switches") @@ -107,4 +108,4 @@ def check_not_retained_switches_between_nodes(original_data, open_not_retained_s open_switches.loc[:, 'VALUE'] = 'true' original_data = triplets.rdf_parser.update_triplet_from_triplet(original_data, open_switches) - return original_data, updated_switches + return original_data, violated_switches diff --git a/emf/model_validator/worker.py b/emf/model_validator/worker.py index 219111ca..80fce5c8 100644 --- a/emf/model_validator/worker.py +++ b/emf/model_validator/worker.py @@ -19,6 +19,7 @@ # RabbitMQ consumer implementation consumer = rabbit.RMQConsumer(queue=INPUT_RMQ_QUEUE, message_handlers=[HandlerModelsValidator()], + forward=OUTPUT_RMQ_EXCHANGE, ) try: diff --git a/emf/task_generator/manual_worker.py b/emf/task_generator/manual_worker.py index 98810c6c..fdb7baa2 100644 --- a/emf/task_generator/manual_worker.py +++ b/emf/task_generator/manual_worker.py @@ -1,8 +1,10 @@ import json import os +import sys import config import logging -from emf.task_generator.task_generator import generate_tasks, filter_and_flatten_dict +from emf.task_generator.task_generator import generate_tasks +from emf.common.helpers.utils import filter_and_flatten_dict from emf.common.integrations import rabbit from emf.common.config_parser import parse_app_properties from emf.common.logging.custom_logger import initialize_custom_logger @@ -23,13 +25,13 @@ # Based on RUN_TYPE get process timeframe config -break_top=False +break_top = False for merge_index, merge_type in enumerate(process_config_json): for run in merge_type.get("runs", []): if RUN_TYPE in run["@id"]: process_config_json.pop(merge_index) process_config_json[0]['runs'] = [run] - break_top=True + break_top = True break if break_top: break @@ -39,6 +41,11 @@ timeframe_config_json = [time_conf] break +if "RMM" in RUN_TYPE and not(INCLUDED_TSO): + logger.error(f"RMM included TSOs can not be empty for the run type: {RUN_TYPE}") + sys.exit("Issue with input, check the EMFOS logs for possible error") + + # Update process configuration from ENV variables if defined process_config_json[0]['runs'][0]['run_at'] = '* * * * *' process_config_json[0]['runs'][0]['properties']['included'] = [tso.strip() for tso in INCLUDED_TSO.split(',')] if INCLUDED_TSO else [] @@ -47,12 +54,10 @@ process_config_json[0]['runs'][0]['properties']['fix_net_interchange2'] = FIX_NET_INTERCHANGE2 process_config_json[0]['runs'][0]['properties']['version'] = TASK_VERSION process_config_json[0]['runs'][0]['properties']['replacement'] = RUN_REPLACEMENT -process_config_json[0]['runs'][0]['properties']['replacement_local'] = RUN_REPLACEMENT_LOCAL process_config_json[0]['runs'][0]['properties']['scaling'] = RUN_SCALING process_config_json[0]['runs'][0]['properties']['upload_to_opdm'] = UPLOAD_TO_OPDM process_config_json[0]['runs'][0]['properties']['upload_to_minio'] = UPLOAD_TO_MINIO process_config_json[0]['runs'][0]['properties']['send_merge_report'] = SEND_MERGE_REPORT -process_config_json[0]['runs'][0]['properties']['pre_temp_fixes'] = PRE_TEMP_FIXES process_config_json[0]['runs'][0]['properties']['post_temp_fixes'] = POST_TEMP_FIXES process_config_json[0]['runs'][0]['properties']['force_outage_fix'] = FORCE_OUTAGE_FIX diff --git a/emf/task_generator/task_generator.py b/emf/task_generator/task_generator.py index 6f2fd133..b092bd06 100644 --- a/emf/task_generator/task_generator.py +++ b/emf/task_generator/task_generator.py @@ -1,17 +1,15 @@ import aniso8601 from datetime import datetime - -from emf.common.time_helper import parse_duration, convert_to_utc, convert_to_timezone, timezone, reference_times, utcnow -from emf.common.integrations.object_storage.tasks import publish_tasks -from emf.common.integrations.object_storage.models import query_data -from emf.common.config_parser import parse_app_properties import config from uuid import uuid4 import croniter -from pathlib import Path import json import logging -from os import getlogin +import os +from emf.common.helpers.time import parse_duration, convert_to_utc, convert_to_timezone, timezone, reference_times, utcnow +from emf.common.helpers.tasks import update_task_status +from emf.common.integrations.elastic import Elastic +from emf.common.config_parser import parse_app_properties logger = logging.getLogger(__name__) parse_app_properties(globals(), config.paths.task_generator.task_generator) @@ -51,7 +49,6 @@ def generate_tasks(task_window_duration: str, time_frames = {time_frame["@id"].split("/")[-1]: time_frame for time_frame in timeframe_conf} # Load the process configuration from the specified file. - for process in process_conf: # Loop through each run in the process configuration. @@ -65,7 +62,7 @@ def generate_tasks(task_window_duration: str, if timetravel_now: now = aniso8601.parse_datetime(timetravel_now) - logger.info(f"Now -> {now}") + logger.info(f"Now: {now}") # Calculate the start and end of the task window for the run. run_window_start = reference_times[task_window_reference](now) @@ -84,20 +81,20 @@ def generate_tasks(task_window_duration: str, else: _ = runs.get_next(datetime) - logger.info(f"Next run of {run['@id']} at {run_timestamp}") + logger.info(f"Next run of {run['@id']} at: {run_timestamp}") - if not (run_timestamp >= run_window_start and run_timestamp <= run_window_end): - logger.info(f"Run at {run_timestamp} not in window [{run_window_start}/{run_window_end}] -> {run['@id']}") + if not (run_window_start <= run_timestamp <= run_window_end): + logger.info(f"Run at {run_timestamp} not in window [{run_window_start}/{run_window_end}]: {run['@id']}") # Loop through each timestamp in the current run. while run_timestamp <= run_window_end: - logger.info(f"Run at {run_timestamp} in window [{run_window_start}/{run_window_end}] -> {run['@id']}") + logger.info(f"Run at {run_timestamp} in window [{run_window_start}/{run_window_end}]: {run['@id']}") # Get the reference time for the current timestamp in the time frame. reference_time_start = reference_times[time_frame["reference_time_start"]](run_timestamp) reference_time_end = reference_times[time_frame["reference_time_end"]](run_timestamp) - # Change reference time according to time shift config + # Change reference time according to time shift config. if process_time_shift: reference_time_start = reference_time_start + parse_duration(process_time_shift) reference_time_end = reference_time_end + parse_duration(process_time_shift) @@ -110,7 +107,7 @@ def generate_tasks(task_window_duration: str, job_period_start_utc = convert_to_utc(job_period_start) job_period_end_utc = convert_to_utc(job_period_end) - # Calculate the open and close times for the gate (the window of time before the job period start time during wich the job/tasks must be done). + # Calculate the open and close times for the gate (the window of time before the job period start time during which the job/tasks must be done). gate_open_utc = job_period_start_utc - parse_duration(run["gate_open"]) gate_close_utc = job_period_start_utc - parse_duration(run["gate_close"]) @@ -133,7 +130,7 @@ def generate_tasks(task_window_duration: str, task_id = str(uuid4()) task_timestamp = utcnow().isoformat() - logger.info(f"Task {timestamp_utc} in window [{job_period_start_utc}/{job_period_end_utc}] -> Job: {job_id}") + logger.info(f"Task {timestamp_utc} in window [{job_period_start_utc}/{job_period_end_utc}] for job: {job_id}") task = { "@context": "https://example.com/task_context.jsonld", @@ -143,7 +140,7 @@ def generate_tasks(task_window_duration: str, "run_id": run.get("@id", None), "job_id": f"urn:uuid:{job_id}", "task_type": "automatic", - "task_initiator": getlogin(), + "task_initiator": os.environ.get("USERNAME", "unknown"), "task_priority": run.get("priority", process.get("priority", "normal")), # "low", "normal", "high" "task_creation_time": task_timestamp, "task_update_time": "", @@ -174,13 +171,10 @@ def generate_tasks(task_window_duration: str, task["task_tags"].extend(run.get("tags", [])) # Check if task already exists, then set version number accordingly - if TASK_ELK_INDEX: - set_task_version(task, TASK_ELK_INDEX) - else: - set_task_version(task) + set_task_version(task=task) # Update task status - update_task_status(task, "created") + update_task_status(task=task, status_text="created") # Return Task yield task @@ -194,96 +188,53 @@ def generate_tasks(task_window_duration: str, run_timestamp = runs.get_next(datetime) -def flatten_dict(nested_dict: dict, parent_key: str = '', separator: str = '.'): - """ - Flattens a nested dictionary. - - Parameters: - - nested_dict (dict): The dictionary to flatten. - - parent_key (str): The base key string used for recursion. - - separator (str): The separator between parent and child keys. - - Returns: - - dict: A flattened dictionary where nested keys are concatenated into a single string. - """ - items = [] - for k, v in nested_dict.items(): - new_key = f"{parent_key}{separator}{k}" if parent_key else k - if isinstance(v, dict): - items.extend(flatten_dict(v, new_key, separator=separator).items()) - elif isinstance(v, list): - for i, item in enumerate(v): - if isinstance(item, dict): - items.extend(flatten_dict(item, f"{new_key}[{i}]", separator=separator).items()) - else: - items.append((f"{new_key}[{i}]", item)) - else: - items.append((new_key, v)) - return dict(items) - - -def filter_and_flatten_dict(nested_dict: dict, keys: list): - """ - Creates a new flat dictionary from specified keys. - - Parameters: - - nested_dict (dict): The original nested dictionary. - - keys (list): The list of keys to include in the new flat dictionary. - - Returns: - - dict: A new flat dictionary with only the specified keys. - """ - flattened = flatten_dict(nested_dict) - return {key: flattened[key] for key in keys if key in flattened} - - -def update_task_status(task, status_text, publish=True): - """Update task status - Will update task_update_time - Will update task_status - Will append new status to task_status_trace""" - - logger.info(f"Updating Task status to -> {status_text}") - - utc_now = datetime.utcnow().isoformat() - - task["task_update_time"] = utc_now - task["task_status"] = status_text +def set_task_version(task: dict): - task["task_status_trace"].append({ - "status": status_text, - "timestamp": utc_now - }) + # Check versioning mode + auto_versioning_enabled = False + if task['task_properties']['version'] == 'AUTO': + logger.debug("Task versioning set to AUTO mode") + auto_versioning_enabled = True - # TODO - better handling if elk is not available, possibly set elk connection timout really small or refactro the sending to happen via rabbit - if publish: - try: - publish_tasks([task]) - except: - logger.warning("Task publication to Elastic failed") - - -def set_task_version(task, elk_index='emfos-tasks*'): - query = {'task_properties.timestamp_utc': task['task_properties']['timestamp_utc'], - 'task_properties.time_horizon': task['task_properties']['time_horizon'], - 'task_properties.merge_type': task['task_properties']['merge_type']} + query = { + "bool": { + "must": [ + {"match": {"task_properties.timestamp_utc": task['task_properties']['timestamp_utc']}}, + {"term": {"task_properties.time_horizon.keyword": task['task_properties']['time_horizon']}}, + {"term": {"task_properties.merge_type.keyword": task['task_properties']['merge_type']}}, + ] + } + } + service = Elastic() try: - task_list = query_data(query, index=elk_index) - if task_list: - latest_version = max(int(item['task_properties'].get('version', "0")) for item in task_list if item['task_properties'].get('version', 1)) - if task['task_properties']['version'] == 'AUTO': - task['task_properties']['version'] = str(int(latest_version) + 1).zfill(3) - elif int(latest_version) > int(task['task_properties']['version']): - task['task_properties']['version'] = str(int(latest_version) + 1).zfill(3) - else: - logger.error("Version set lower than existing model") + updated_version = None + tasks_df = service.get_docs_by_query(index=TASK_ELK_INDEX, query=query) + + if tasks_df.empty: + logger.info(f"No previous runs found for task, using version from configuration: {task['task_properties']['version']}") + if auto_versioning_enabled: + logger.info("Task versioning mode 'AUTO', defaulting to: '001'") + updated_version = '001' else: - if task['task_properties']['version'] == 'AUTO': - task['task_properties']['version'] = '001' + latest_version = tasks_df['task_properties.version'].max() + logger.info(f"Latest available task version: {latest_version}") + if auto_versioning_enabled: + updated_version = str(int(latest_version) + 1).zfill(3) + elif int(latest_version) >= int(task['task_properties']['version']): + logger.warning("Latest available version is equal or higher than defined in task, increasing from latest") + updated_version = str(int(latest_version) + 1).zfill(3) + else: + logger.info("Using version for task configuration") + updated_version = task['task_properties']['version'] + + if updated_version: + task['task_properties']['version'] = updated_version + logger.info(f"Version set to: {updated_version}") - except: + except Exception as e: logger.warning("Elastic query for task versioning unsuccessful, version not updated") + logger.warning(f"Exception traceback: {e}") if __name__ == "__main__": @@ -297,10 +248,13 @@ def set_task_version(task, elk_index='emfos-tasks*'): task_window_duration = "P1D" task_window_reference = "currentDayStart" - timeframe_conf = "../../config/task_generator/timeframe_conf.json" - process_conf = "../../config/task_generator/process_conf.json" + timeframe_conf = config.paths.task_generator.timeframe_conf + process_conf = config.paths.task_generator.process_conf + + timeframe_config_json = json.load(timeframe_conf) + process_config_json = json.load(process_conf) - tasks = list(generate_tasks(task_window_duration, task_window_reference, process_conf, timeframe_conf)) + tasks = list(generate_tasks(task_window_duration, task_window_reference, process_config_json, timeframe_config_json)) tasks_table = pandas.json_normalize(tasks) print(tasks_table["process_id"].value_counts()) diff --git a/emf/task_generator/worker.py b/emf/task_generator/worker.py index c74485ac..c4789e32 100644 --- a/emf/task_generator/worker.py +++ b/emf/task_generator/worker.py @@ -1,7 +1,8 @@ import json import config import logging -from emf.task_generator.task_generator import generate_tasks, filter_and_flatten_dict +from emf.task_generator.task_generator import generate_tasks +from emf.common.helpers.utils import filter_and_flatten_dict from emf.common.integrations import rabbit from emf.common.config_parser import parse_app_properties from emf.common.logging.custom_logger import initialize_custom_logger @@ -23,13 +24,13 @@ if tasks: logger.info(f"Creating connection to RMQ") rabbit_service = rabbit.BlockingClient() - logger.info(f"Sending tasks to Rabbit exchange '{RMQ_EXCHANGE}'") + logger.info(f"Sending tasks to Rabbit exchange: {RMQ_EXCHANGE}") for task in tasks: elk_handler.start_trace(task) - rabbit_service.publish(payload=json.dumps(task), exchange_name=RMQ_EXCHANGE, headers=filter_and_flatten_dict(task, TASK_HEADER_KEYS.split(","))) + rabbit_service.publish(payload=json.dumps(task), + exchange_name=RMQ_EXCHANGE, + headers=filter_and_flatten_dict(task, TASK_HEADER_KEYS.split(","))) elk_handler.stop_trace() else: - logger.info("No tasks generated at current time.") - - + logger.info("No tasks generated at current timeframe, exiting worker.") diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..98da0e93 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,34 @@ +[project] +name = "emf" +version = "0.1.0" +description = "Add your description here" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "aniso8601>=10.0.1", + "croniter>=6.0.0", + "edx>=0.0.16", + "elasticsearch>=9.0.1", + "lxml>=5.4.0", + "minio>=7.2.15", + "ndjson>=0.3.1", + "opdm-api>=0.1.3", + "pandas==2.2.2", + "pika>=1.3.2", + "pypowsybl==1.9.0", + "pytz>=2025.2", + "requests>=2.32.3", + "saxonche>=12.7.0", + "triplets>=0.0.10", +] + +[[tool.uv.index]] +name = "pypi" +url = "https://pypi.org/simple" + +[[tool.uv.index]] +name = "rcc-pypi" +url = "https://artifactory.elering.sise/artifactory/api/pypi/rcc-pypi/simple" + +[tool.uv] +allow-insecure-host = ["artifactory.elering.sise"] diff --git a/requirements.txt b/requirements.txt index 5a31a1b9..a43f05b8 100644 Binary files a/requirements.txt and b/requirements.txt differ diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..02506243 --- /dev/null +++ b/uv.lock @@ -0,0 +1,718 @@ +version = 1 +revision = 2 +requires-python = ">=3.11" +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version < '3.12'", +] + +[[package]] +name = "aniso8601" +version = "10.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8b/8d/52179c4e3f1978d3d9a285f98c706642522750ef343e9738286130423730/aniso8601-10.0.1.tar.gz", hash = "sha256:25488f8663dd1528ae1f54f94ac1ea51ae25b4d531539b8bc707fed184d16845", size = 47190, upload-time = "2025-04-18T17:29:42.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/59/75/e0e10dc7ed1408c28e03a6cb2d7a407f99320eb953f229d008a7a6d05546/aniso8601-10.0.1-py2.py3-none-any.whl", hash = "sha256:eb19717fd4e0db6de1aab06f12450ab92144246b257423fe020af5748c0cb89e", size = 52848, upload-time = "2025-04-18T17:29:41.492Z" }, +] + +[[package]] +name = "argon2-cffi" +version = "25.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi-bindings" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/89/ce5af8a7d472a67cc819d5d998aa8c82c5d860608c4db9f46f1162d7dab9/argon2_cffi-25.1.0.tar.gz", hash = "sha256:694ae5cc8a42f4c4e2bf2ca0e64e51e23a040c6a517a85074683d3959e1346c1", size = 45706, upload-time = "2025-06-03T06:55:32.073Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/d3/a8b22fa575b297cd6e3e3b0155c7e25db170edf1c74783d6a31a2490b8d9/argon2_cffi-25.1.0-py3-none-any.whl", hash = "sha256:fdc8b074db390fccb6eb4a3604ae7231f219aa669a2652e0f20e16ba513d5741", size = 14657, upload-time = "2025-06-03T06:55:30.804Z" }, +] + +[[package]] +name = "argon2-cffi-bindings" +version = "21.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/e9/184b8ccce6683b0aa2fbb7ba5683ea4b9c5763f1356347f1312c32e3c66e/argon2-cffi-bindings-21.2.0.tar.gz", hash = "sha256:bb89ceffa6c791807d1305ceb77dbfacc5aa499891d2c55661c6459651fc39e3", size = 1779911, upload-time = "2021-12-01T08:52:55.68Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d4/13/838ce2620025e9666aa8f686431f67a29052241692a3dd1ae9d3692a89d3/argon2_cffi_bindings-21.2.0-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ccb949252cb2ab3a08c02024acb77cfb179492d5701c7cbdbfd776124d4d2367", size = 29658, upload-time = "2021-12-01T09:09:17.016Z" }, + { url = "https://files.pythonhosted.org/packages/b3/02/f7f7bb6b6af6031edb11037639c697b912e1dea2db94d436e681aea2f495/argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9524464572e12979364b7d600abf96181d3541da11e23ddf565a32e70bd4dc0d", size = 80583, upload-time = "2021-12-01T09:09:19.546Z" }, + { url = "https://files.pythonhosted.org/packages/ec/f7/378254e6dd7ae6f31fe40c8649eea7d4832a42243acaf0f1fff9083b2bed/argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b746dba803a79238e925d9046a63aa26bf86ab2a2fe74ce6b009a1c3f5c8f2ae", size = 86168, upload-time = "2021-12-01T09:09:21.445Z" }, + { url = "https://files.pythonhosted.org/packages/74/f6/4a34a37a98311ed73bb80efe422fed95f2ac25a4cacc5ae1d7ae6a144505/argon2_cffi_bindings-21.2.0-cp36-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:58ed19212051f49a523abb1dbe954337dc82d947fb6e5a0da60f7c8471a8476c", size = 82709, upload-time = "2021-12-01T09:09:18.182Z" }, + { url = "https://files.pythonhosted.org/packages/74/2b/73d767bfdaab25484f7e7901379d5f8793cccbb86c6e0cbc4c1b96f63896/argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:bd46088725ef7f58b5a1ef7ca06647ebaf0eb4baff7d1d0d177c6cc8744abd86", size = 83613, upload-time = "2021-12-01T09:09:22.741Z" }, + { url = "https://files.pythonhosted.org/packages/4f/fd/37f86deef67ff57c76f137a67181949c2d408077e2e3dd70c6c42912c9bf/argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_i686.whl", hash = "sha256:8cd69c07dd875537a824deec19f978e0f2078fdda07fd5c42ac29668dda5f40f", size = 84583, upload-time = "2021-12-01T09:09:24.177Z" }, + { url = "https://files.pythonhosted.org/packages/6f/52/5a60085a3dae8fded8327a4f564223029f5f54b0cb0455a31131b5363a01/argon2_cffi_bindings-21.2.0-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:f1152ac548bd5b8bcecfb0b0371f082037e47128653df2e8ba6e914d384f3c3e", size = 88475, upload-time = "2021-12-01T09:09:26.673Z" }, + { url = "https://files.pythonhosted.org/packages/8b/95/143cd64feb24a15fa4b189a3e1e7efbaeeb00f39a51e99b26fc62fbacabd/argon2_cffi_bindings-21.2.0-cp36-abi3-win32.whl", hash = "sha256:603ca0aba86b1349b147cab91ae970c63118a0f30444d4bc80355937c950c082", size = 27698, upload-time = "2021-12-01T09:09:27.87Z" }, + { url = "https://files.pythonhosted.org/packages/37/2c/e34e47c7dee97ba6f01a6203e0383e15b60fb85d78ac9a15cd066f6fe28b/argon2_cffi_bindings-21.2.0-cp36-abi3-win_amd64.whl", hash = "sha256:b2ef1c30440dbbcba7a5dc3e319408b59676e2e039e2ae11a8775ecf482b192f", size = 30817, upload-time = "2021-12-01T09:09:30.267Z" }, + { url = "https://files.pythonhosted.org/packages/5a/e4/bf8034d25edaa495da3c8a3405627d2e35758e44ff6eaa7948092646fdcc/argon2_cffi_bindings-21.2.0-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:e415e3f62c8d124ee16018e491a009937f8cf7ebf5eb430ffc5de21b900dad93", size = 53104, upload-time = "2021-12-01T09:09:31.335Z" }, +] + +[[package]] +name = "attrs" +version = "25.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/b0/1367933a8532ee6ff8d63537de4f1177af4bff9f3e829baf7331f595bb24/attrs-25.3.0.tar.gz", hash = "sha256:75d7cefc7fb576747b2c81b4442d4d4a1ce0900973527c011d1030fd3bf4af1b", size = 812032, upload-time = "2025-03-13T11:10:22.779Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/06/bb80f5f86020c4551da315d78b3ab75e8228f89f0162f2c3a819e407941a/attrs-25.3.0-py3-none-any.whl", hash = "sha256:427318ce031701fea540783410126f03899a97ffc6f61596ad581ac2e40e3bc3", size = 63815, upload-time = "2025-03-13T11:10:21.14Z" }, +] + +[[package]] +name = "certifi" +version = "2025.4.26" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/9e/c05b3920a3b7d20d3d3310465f50348e5b3694f4f88c6daf736eef3024c4/certifi-2025.4.26.tar.gz", hash = "sha256:0a816057ea3cdefcef70270d2c515e4506bbc954f417fa5ade2021213bb8f0c6", size = 160705, upload-time = "2025-04-26T02:12:29.51Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4a/7e/3db2bd1b1f9e95f7cddca6d6e75e2f2bd9f51b1246e546d88addca0106bd/certifi-2025.4.26-py3-none-any.whl", hash = "sha256:30350364dfe371162649852c63336a15c70c6510c2ad5015b21c2345311805f3", size = 159618, upload-time = "2025-04-26T02:12:27.662Z" }, +] + +[[package]] +name = "cffi" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/f4/927e3a8899e52a27fa57a48607ff7dc91a9ebe97399b357b85a0c7892e00/cffi-1.17.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a45e3c6913c5b87b3ff120dcdc03f6131fa0065027d0ed7ee6190736a74cd401", size = 182264, upload-time = "2024-09-04T20:43:51.124Z" }, + { url = "https://files.pythonhosted.org/packages/6c/f5/6c3a8efe5f503175aaddcbea6ad0d2c96dad6f5abb205750d1b3df44ef29/cffi-1.17.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:30c5e0cb5ae493c04c8b42916e52ca38079f1b235c2f8ae5f4527b963c401caf", size = 178651, upload-time = "2024-09-04T20:43:52.872Z" }, + { url = "https://files.pythonhosted.org/packages/94/dd/a3f0118e688d1b1a57553da23b16bdade96d2f9bcda4d32e7d2838047ff7/cffi-1.17.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f75c7ab1f9e4aca5414ed4d8e5c0e303a34f4421f8a0d47a4d019ceff0ab6af4", size = 445259, upload-time = "2024-09-04T20:43:56.123Z" }, + { url = "https://files.pythonhosted.org/packages/2e/ea/70ce63780f096e16ce8588efe039d3c4f91deb1dc01e9c73a287939c79a6/cffi-1.17.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1ed2dd2972641495a3ec98445e09766f077aee98a1c896dcb4ad0d303628e41", size = 469200, upload-time = "2024-09-04T20:43:57.891Z" }, + { url = "https://files.pythonhosted.org/packages/1c/a0/a4fa9f4f781bda074c3ddd57a572b060fa0df7655d2a4247bbe277200146/cffi-1.17.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:46bf43160c1a35f7ec506d254e5c890f3c03648a4dbac12d624e4490a7046cd1", size = 477235, upload-time = "2024-09-04T20:44:00.18Z" }, + { url = "https://files.pythonhosted.org/packages/62/12/ce8710b5b8affbcdd5c6e367217c242524ad17a02fe5beec3ee339f69f85/cffi-1.17.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a24ed04c8ffd54b0729c07cee15a81d964e6fee0e3d4d342a27b020d22959dc6", size = 459721, upload-time = "2024-09-04T20:44:01.585Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6b/d45873c5e0242196f042d555526f92aa9e0c32355a1be1ff8c27f077fd37/cffi-1.17.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:610faea79c43e44c71e1ec53a554553fa22321b65fae24889706c0a84d4ad86d", size = 467242, upload-time = "2024-09-04T20:44:03.467Z" }, + { url = "https://files.pythonhosted.org/packages/1a/52/d9a0e523a572fbccf2955f5abe883cfa8bcc570d7faeee06336fbd50c9fc/cffi-1.17.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a9b15d491f3ad5d692e11f6b71f7857e7835eb677955c00cc0aefcd0669adaf6", size = 477999, upload-time = "2024-09-04T20:44:05.023Z" }, + { url = "https://files.pythonhosted.org/packages/44/74/f2a2460684a1a2d00ca799ad880d54652841a780c4c97b87754f660c7603/cffi-1.17.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:de2ea4b5833625383e464549fec1bc395c1bdeeb5f25c4a3a82b5a8c756ec22f", size = 454242, upload-time = "2024-09-04T20:44:06.444Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4a/34599cac7dfcd888ff54e801afe06a19c17787dfd94495ab0c8d35fe99fb/cffi-1.17.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:fc48c783f9c87e60831201f2cce7f3b2e4846bf4d8728eabe54d60700b318a0b", size = 478604, upload-time = "2024-09-04T20:44:08.206Z" }, + { url = "https://files.pythonhosted.org/packages/34/33/e1b8a1ba29025adbdcda5fb3a36f94c03d771c1b7b12f726ff7fef2ebe36/cffi-1.17.1-cp311-cp311-win32.whl", hash = "sha256:85a950a4ac9c359340d5963966e3e0a94a676bd6245a4b55bc43949eee26a655", size = 171727, upload-time = "2024-09-04T20:44:09.481Z" }, + { url = "https://files.pythonhosted.org/packages/3d/97/50228be003bb2802627d28ec0627837ac0bf35c90cf769812056f235b2d1/cffi-1.17.1-cp311-cp311-win_amd64.whl", hash = "sha256:caaf0640ef5f5517f49bc275eca1406b0ffa6aa184892812030f04c2abf589a0", size = 181400, upload-time = "2024-09-04T20:44:10.873Z" }, + { url = "https://files.pythonhosted.org/packages/5a/84/e94227139ee5fb4d600a7a4927f322e1d4aea6fdc50bd3fca8493caba23f/cffi-1.17.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:805b4371bf7197c329fcb3ead37e710d1bca9da5d583f5073b799d5c5bd1eee4", size = 183178, upload-time = "2024-09-04T20:44:12.232Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/fb72c2b48656111c4ef27f0f91da355e130a923473bf5ee75c5643d00cca/cffi-1.17.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:733e99bc2df47476e3848417c5a4540522f234dfd4ef3ab7fafdf555b082ec0c", size = 178840, upload-time = "2024-09-04T20:44:13.739Z" }, + { url = "https://files.pythonhosted.org/packages/cc/b6/db007700f67d151abadf508cbfd6a1884f57eab90b1bb985c4c8c02b0f28/cffi-1.17.1-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1257bdabf294dceb59f5e70c64a3e2f462c30c7ad68092d01bbbfb1c16b1ba36", size = 454803, upload-time = "2024-09-04T20:44:15.231Z" }, + { url = "https://files.pythonhosted.org/packages/1a/df/f8d151540d8c200eb1c6fba8cd0dfd40904f1b0682ea705c36e6c2e97ab3/cffi-1.17.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:da95af8214998d77a98cc14e3a3bd00aa191526343078b530ceb0bd710fb48a5", size = 478850, upload-time = "2024-09-04T20:44:17.188Z" }, + { url = "https://files.pythonhosted.org/packages/28/c0/b31116332a547fd2677ae5b78a2ef662dfc8023d67f41b2a83f7c2aa78b1/cffi-1.17.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d63afe322132c194cf832bfec0dc69a99fb9bb6bbd550f161a49e9e855cc78ff", size = 485729, upload-time = "2024-09-04T20:44:18.688Z" }, + { url = "https://files.pythonhosted.org/packages/91/2b/9a1ddfa5c7f13cab007a2c9cc295b70fbbda7cb10a286aa6810338e60ea1/cffi-1.17.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f79fc4fc25f1c8698ff97788206bb3c2598949bfe0fef03d299eb1b5356ada99", size = 471256, upload-time = "2024-09-04T20:44:20.248Z" }, + { url = "https://files.pythonhosted.org/packages/b2/d5/da47df7004cb17e4955df6a43d14b3b4ae77737dff8bf7f8f333196717bf/cffi-1.17.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b62ce867176a75d03a665bad002af8e6d54644fad99a3c70905c543130e39d93", size = 479424, upload-time = "2024-09-04T20:44:21.673Z" }, + { url = "https://files.pythonhosted.org/packages/0b/ac/2a28bcf513e93a219c8a4e8e125534f4f6db03e3179ba1c45e949b76212c/cffi-1.17.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:386c8bf53c502fff58903061338ce4f4950cbdcb23e2902d86c0f722b786bbe3", size = 484568, upload-time = "2024-09-04T20:44:23.245Z" }, + { url = "https://files.pythonhosted.org/packages/d4/38/ca8a4f639065f14ae0f1d9751e70447a261f1a30fa7547a828ae08142465/cffi-1.17.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:4ceb10419a9adf4460ea14cfd6bc43d08701f0835e979bf821052f1805850fe8", size = 488736, upload-time = "2024-09-04T20:44:24.757Z" }, + { url = "https://files.pythonhosted.org/packages/86/c5/28b2d6f799ec0bdecf44dced2ec5ed43e0eb63097b0f58c293583b406582/cffi-1.17.1-cp312-cp312-win32.whl", hash = "sha256:a08d7e755f8ed21095a310a693525137cfe756ce62d066e53f502a83dc550f65", size = 172448, upload-time = "2024-09-04T20:44:26.208Z" }, + { url = "https://files.pythonhosted.org/packages/50/b9/db34c4755a7bd1cb2d1603ac3863f22bcecbd1ba29e5ee841a4bc510b294/cffi-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:51392eae71afec0d0c8fb1a53b204dbb3bcabcb3c9b807eedf3e1e6ccf2de903", size = 181976, upload-time = "2024-09-04T20:44:27.578Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f8/dd6c246b148639254dad4d6803eb6a54e8c85c6e11ec9df2cffa87571dbe/cffi-1.17.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f3a2b4222ce6b60e2e8b337bb9596923045681d71e5a082783484d845390938e", size = 182989, upload-time = "2024-09-04T20:44:28.956Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f1/672d303ddf17c24fc83afd712316fda78dc6fce1cd53011b839483e1ecc8/cffi-1.17.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0984a4925a435b1da406122d4d7968dd861c1385afe3b45ba82b750f229811e2", size = 178802, upload-time = "2024-09-04T20:44:30.289Z" }, + { url = "https://files.pythonhosted.org/packages/0e/2d/eab2e858a91fdff70533cab61dcff4a1f55ec60425832ddfdc9cd36bc8af/cffi-1.17.1-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d01b12eeeb4427d3110de311e1774046ad344f5b1a7403101878976ecd7a10f3", size = 454792, upload-time = "2024-09-04T20:44:32.01Z" }, + { url = "https://files.pythonhosted.org/packages/75/b2/fbaec7c4455c604e29388d55599b99ebcc250a60050610fadde58932b7ee/cffi-1.17.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:706510fe141c86a69c8ddc029c7910003a17353970cff3b904ff0686a5927683", size = 478893, upload-time = "2024-09-04T20:44:33.606Z" }, + { url = "https://files.pythonhosted.org/packages/4f/b7/6e4a2162178bf1935c336d4da8a9352cccab4d3a5d7914065490f08c0690/cffi-1.17.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de55b766c7aa2e2a3092c51e0483d700341182f08e67c63630d5b6f200bb28e5", size = 485810, upload-time = "2024-09-04T20:44:35.191Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8a/1d0e4a9c26e54746dc08c2c6c037889124d4f59dffd853a659fa545f1b40/cffi-1.17.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c59d6e989d07460165cc5ad3c61f9fd8f1b4796eacbd81cee78957842b834af4", size = 471200, upload-time = "2024-09-04T20:44:36.743Z" }, + { url = "https://files.pythonhosted.org/packages/26/9f/1aab65a6c0db35f43c4d1b4f580e8df53914310afc10ae0397d29d697af4/cffi-1.17.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd398dbc6773384a17fe0d3e7eeb8d1a21c2200473ee6806bb5e6a8e62bb73dd", size = 479447, upload-time = "2024-09-04T20:44:38.492Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e4/fb8b3dd8dc0e98edf1135ff067ae070bb32ef9d509d6cb0f538cd6f7483f/cffi-1.17.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3edc8d958eb099c634dace3c7e16560ae474aa3803a5df240542b305d14e14ed", size = 484358, upload-time = "2024-09-04T20:44:40.046Z" }, + { url = "https://files.pythonhosted.org/packages/f1/47/d7145bf2dc04684935d57d67dff9d6d795b2ba2796806bb109864be3a151/cffi-1.17.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:72e72408cad3d5419375fc87d289076ee319835bdfa2caad331e377589aebba9", size = 488469, upload-time = "2024-09-04T20:44:41.616Z" }, + { url = "https://files.pythonhosted.org/packages/bf/ee/f94057fa6426481d663b88637a9a10e859e492c73d0384514a17d78ee205/cffi-1.17.1-cp313-cp313-win32.whl", hash = "sha256:e03eab0a8677fa80d646b5ddece1cbeaf556c313dcfac435ba11f107ba117b5d", size = 172475, upload-time = "2024-09-04T20:44:43.733Z" }, + { url = "https://files.pythonhosted.org/packages/7c/fc/6a8cb64e5f0324877d503c854da15d76c1e50eb722e320b15345c4d0c6de/cffi-1.17.1-cp313-cp313-win_amd64.whl", hash = "sha256:f6a16c31041f09ead72d69f583767292f750d24913dadacf5756b966aacb3f1a", size = 182009, upload-time = "2024-09-04T20:44:45.309Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/33/89c2ced2b67d1c2a61c19c6751aa8902d46ce3dacb23600a283619f5a12d/charset_normalizer-3.4.2.tar.gz", hash = "sha256:5baececa9ecba31eff645232d59845c07aa030f0c81ee70184a90d35099a0e63", size = 126367, upload-time = "2025-05-02T08:34:42.01Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/85/4c40d00dcc6284a1c1ad5de5e0996b06f39d8232f1031cd23c2f5c07ee86/charset_normalizer-3.4.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:be1e352acbe3c78727a16a455126d9ff83ea2dfdcbc83148d2982305a04714c2", size = 198794, upload-time = "2025-05-02T08:32:11.945Z" }, + { url = "https://files.pythonhosted.org/packages/41/d9/7a6c0b9db952598e97e93cbdfcb91bacd89b9b88c7c983250a77c008703c/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa88ca0b1932e93f2d961bf3addbb2db902198dca337d88c89e1559e066e7645", size = 142846, upload-time = "2025-05-02T08:32:13.946Z" }, + { url = "https://files.pythonhosted.org/packages/66/82/a37989cda2ace7e37f36c1a8ed16c58cf48965a79c2142713244bf945c89/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d524ba3f1581b35c03cb42beebab4a13e6cdad7b36246bd22541fa585a56cccd", size = 153350, upload-time = "2025-05-02T08:32:15.873Z" }, + { url = "https://files.pythonhosted.org/packages/df/68/a576b31b694d07b53807269d05ec3f6f1093e9545e8607121995ba7a8313/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28a1005facc94196e1fb3e82a3d442a9d9110b8434fc1ded7a24a2983c9888d8", size = 145657, upload-time = "2025-05-02T08:32:17.283Z" }, + { url = "https://files.pythonhosted.org/packages/92/9b/ad67f03d74554bed3aefd56fe836e1623a50780f7c998d00ca128924a499/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fdb20a30fe1175ecabed17cbf7812f7b804b8a315a25f24678bcdf120a90077f", size = 147260, upload-time = "2025-05-02T08:32:18.807Z" }, + { url = "https://files.pythonhosted.org/packages/a6/e6/8aebae25e328160b20e31a7e9929b1578bbdc7f42e66f46595a432f8539e/charset_normalizer-3.4.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0f5d9ed7f254402c9e7d35d2f5972c9bbea9040e99cd2861bd77dc68263277c7", size = 149164, upload-time = "2025-05-02T08:32:20.333Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f2/b3c2f07dbcc248805f10e67a0262c93308cfa149a4cd3d1fe01f593e5fd2/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:efd387a49825780ff861998cd959767800d54f8308936b21025326de4b5a42b9", size = 144571, upload-time = "2025-05-02T08:32:21.86Z" }, + { url = "https://files.pythonhosted.org/packages/60/5b/c3f3a94bc345bc211622ea59b4bed9ae63c00920e2e8f11824aa5708e8b7/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:f0aa37f3c979cf2546b73e8222bbfa3dc07a641585340179d768068e3455e544", size = 151952, upload-time = "2025-05-02T08:32:23.434Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4d/ff460c8b474122334c2fa394a3f99a04cf11c646da895f81402ae54f5c42/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:e70e990b2137b29dc5564715de1e12701815dacc1d056308e2b17e9095372a82", size = 155959, upload-time = "2025-05-02T08:32:24.993Z" }, + { url = "https://files.pythonhosted.org/packages/a2/2b/b964c6a2fda88611a1fe3d4c400d39c66a42d6c169c924818c848f922415/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:0c8c57f84ccfc871a48a47321cfa49ae1df56cd1d965a09abe84066f6853b9c0", size = 153030, upload-time = "2025-05-02T08:32:26.435Z" }, + { url = "https://files.pythonhosted.org/packages/59/2e/d3b9811db26a5ebf444bc0fa4f4be5aa6d76fc6e1c0fd537b16c14e849b6/charset_normalizer-3.4.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6b66f92b17849b85cad91259efc341dce9c1af48e2173bf38a85c6329f1033e5", size = 148015, upload-time = "2025-05-02T08:32:28.376Z" }, + { url = "https://files.pythonhosted.org/packages/90/07/c5fd7c11eafd561bb51220d600a788f1c8d77c5eef37ee49454cc5c35575/charset_normalizer-3.4.2-cp311-cp311-win32.whl", hash = "sha256:daac4765328a919a805fa5e2720f3e94767abd632ae410a9062dff5412bae65a", size = 98106, upload-time = "2025-05-02T08:32:30.281Z" }, + { url = "https://files.pythonhosted.org/packages/a8/05/5e33dbef7e2f773d672b6d79f10ec633d4a71cd96db6673625838a4fd532/charset_normalizer-3.4.2-cp311-cp311-win_amd64.whl", hash = "sha256:e53efc7c7cee4c1e70661e2e112ca46a575f90ed9ae3fef200f2a25e954f4b28", size = 105402, upload-time = "2025-05-02T08:32:32.191Z" }, + { url = "https://files.pythonhosted.org/packages/d7/a4/37f4d6035c89cac7930395a35cc0f1b872e652eaafb76a6075943754f095/charset_normalizer-3.4.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:0c29de6a1a95f24b9a1aa7aefd27d2487263f00dfd55a77719b530788f75cff7", size = 199936, upload-time = "2025-05-02T08:32:33.712Z" }, + { url = "https://files.pythonhosted.org/packages/ee/8a/1a5e33b73e0d9287274f899d967907cd0bf9c343e651755d9307e0dbf2b3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cddf7bd982eaa998934a91f69d182aec997c6c468898efe6679af88283b498d3", size = 143790, upload-time = "2025-05-02T08:32:35.768Z" }, + { url = "https://files.pythonhosted.org/packages/66/52/59521f1d8e6ab1482164fa21409c5ef44da3e9f653c13ba71becdd98dec3/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcbe676a55d7445b22c10967bceaaf0ee69407fbe0ece4d032b6eb8d4565982a", size = 153924, upload-time = "2025-05-02T08:32:37.284Z" }, + { url = "https://files.pythonhosted.org/packages/86/2d/fb55fdf41964ec782febbf33cb64be480a6b8f16ded2dbe8db27a405c09f/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d41c4d287cfc69060fa91cae9683eacffad989f1a10811995fa309df656ec214", size = 146626, upload-time = "2025-05-02T08:32:38.803Z" }, + { url = "https://files.pythonhosted.org/packages/8c/73/6ede2ec59bce19b3edf4209d70004253ec5f4e319f9a2e3f2f15601ed5f7/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4e594135de17ab3866138f496755f302b72157d115086d100c3f19370839dd3a", size = 148567, upload-time = "2025-05-02T08:32:40.251Z" }, + { url = "https://files.pythonhosted.org/packages/09/14/957d03c6dc343c04904530b6bef4e5efae5ec7d7990a7cbb868e4595ee30/charset_normalizer-3.4.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cf713fe9a71ef6fd5adf7a79670135081cd4431c2943864757f0fa3a65b1fafd", size = 150957, upload-time = "2025-05-02T08:32:41.705Z" }, + { url = "https://files.pythonhosted.org/packages/0d/c8/8174d0e5c10ccebdcb1b53cc959591c4c722a3ad92461a273e86b9f5a302/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a370b3e078e418187da8c3674eddb9d983ec09445c99a3a263c2011993522981", size = 145408, upload-time = "2025-05-02T08:32:43.709Z" }, + { url = "https://files.pythonhosted.org/packages/58/aa/8904b84bc8084ac19dc52feb4f5952c6df03ffb460a887b42615ee1382e8/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:a955b438e62efdf7e0b7b52a64dc5c3396e2634baa62471768a64bc2adb73d5c", size = 153399, upload-time = "2025-05-02T08:32:46.197Z" }, + { url = "https://files.pythonhosted.org/packages/c2/26/89ee1f0e264d201cb65cf054aca6038c03b1a0c6b4ae998070392a3ce605/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:7222ffd5e4de8e57e03ce2cef95a4c43c98fcb72ad86909abdfc2c17d227fc1b", size = 156815, upload-time = "2025-05-02T08:32:48.105Z" }, + { url = "https://files.pythonhosted.org/packages/fd/07/68e95b4b345bad3dbbd3a8681737b4338ff2c9df29856a6d6d23ac4c73cb/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:bee093bf902e1d8fc0ac143c88902c3dfc8941f7ea1d6a8dd2bcb786d33db03d", size = 154537, upload-time = "2025-05-02T08:32:49.719Z" }, + { url = "https://files.pythonhosted.org/packages/77/1a/5eefc0ce04affb98af07bc05f3bac9094513c0e23b0562d64af46a06aae4/charset_normalizer-3.4.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:dedb8adb91d11846ee08bec4c8236c8549ac721c245678282dcb06b221aab59f", size = 149565, upload-time = "2025-05-02T08:32:51.404Z" }, + { url = "https://files.pythonhosted.org/packages/37/a0/2410e5e6032a174c95e0806b1a6585eb21e12f445ebe239fac441995226a/charset_normalizer-3.4.2-cp312-cp312-win32.whl", hash = "sha256:db4c7bf0e07fc3b7d89ac2a5880a6a8062056801b83ff56d8464b70f65482b6c", size = 98357, upload-time = "2025-05-02T08:32:53.079Z" }, + { url = "https://files.pythonhosted.org/packages/6c/4f/c02d5c493967af3eda9c771ad4d2bbc8df6f99ddbeb37ceea6e8716a32bc/charset_normalizer-3.4.2-cp312-cp312-win_amd64.whl", hash = "sha256:5a9979887252a82fefd3d3ed2a8e3b937a7a809f65dcb1e068b090e165bbe99e", size = 105776, upload-time = "2025-05-02T08:32:54.573Z" }, + { url = "https://files.pythonhosted.org/packages/ea/12/a93df3366ed32db1d907d7593a94f1fe6293903e3e92967bebd6950ed12c/charset_normalizer-3.4.2-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:926ca93accd5d36ccdabd803392ddc3e03e6d4cd1cf17deff3b989ab8e9dbcf0", size = 199622, upload-time = "2025-05-02T08:32:56.363Z" }, + { url = "https://files.pythonhosted.org/packages/04/93/bf204e6f344c39d9937d3c13c8cd5bbfc266472e51fc8c07cb7f64fcd2de/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eba9904b0f38a143592d9fc0e19e2df0fa2e41c3c3745554761c5f6447eedabf", size = 143435, upload-time = "2025-05-02T08:32:58.551Z" }, + { url = "https://files.pythonhosted.org/packages/22/2a/ea8a2095b0bafa6c5b5a55ffdc2f924455233ee7b91c69b7edfcc9e02284/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3fddb7e2c84ac87ac3a947cb4e66d143ca5863ef48e4a5ecb83bd48619e4634e", size = 153653, upload-time = "2025-05-02T08:33:00.342Z" }, + { url = "https://files.pythonhosted.org/packages/b6/57/1b090ff183d13cef485dfbe272e2fe57622a76694061353c59da52c9a659/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:98f862da73774290f251b9df8d11161b6cf25b599a66baf087c1ffe340e9bfd1", size = 146231, upload-time = "2025-05-02T08:33:02.081Z" }, + { url = "https://files.pythonhosted.org/packages/e2/28/ffc026b26f441fc67bd21ab7f03b313ab3fe46714a14b516f931abe1a2d8/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c9379d65defcab82d07b2a9dfbfc2e95bc8fe0ebb1b176a3190230a3ef0e07c", size = 148243, upload-time = "2025-05-02T08:33:04.063Z" }, + { url = "https://files.pythonhosted.org/packages/c0/0f/9abe9bd191629c33e69e47c6ef45ef99773320e9ad8e9cb08b8ab4a8d4cb/charset_normalizer-3.4.2-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e635b87f01ebc977342e2697d05b56632f5f879a4f15955dfe8cef2448b51691", size = 150442, upload-time = "2025-05-02T08:33:06.418Z" }, + { url = "https://files.pythonhosted.org/packages/67/7c/a123bbcedca91d5916c056407f89a7f5e8fdfce12ba825d7d6b9954a1a3c/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c95a1e2902a8b722868587c0e1184ad5c55631de5afc0eb96bc4b0d738092c0", size = 145147, upload-time = "2025-05-02T08:33:08.183Z" }, + { url = "https://files.pythonhosted.org/packages/ec/fe/1ac556fa4899d967b83e9893788e86b6af4d83e4726511eaaad035e36595/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:ef8de666d6179b009dce7bcb2ad4c4a779f113f12caf8dc77f0162c29d20490b", size = 153057, upload-time = "2025-05-02T08:33:09.986Z" }, + { url = "https://files.pythonhosted.org/packages/2b/ff/acfc0b0a70b19e3e54febdd5301a98b72fa07635e56f24f60502e954c461/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:32fc0341d72e0f73f80acb0a2c94216bd704f4f0bce10aedea38f30502b271ff", size = 156454, upload-time = "2025-05-02T08:33:11.814Z" }, + { url = "https://files.pythonhosted.org/packages/92/08/95b458ce9c740d0645feb0e96cea1f5ec946ea9c580a94adfe0b617f3573/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:289200a18fa698949d2b39c671c2cc7a24d44096784e76614899a7ccf2574b7b", size = 154174, upload-time = "2025-05-02T08:33:13.707Z" }, + { url = "https://files.pythonhosted.org/packages/78/be/8392efc43487ac051eee6c36d5fbd63032d78f7728cb37aebcc98191f1ff/charset_normalizer-3.4.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a476b06fbcf359ad25d34a057b7219281286ae2477cc5ff5e3f70a246971148", size = 149166, upload-time = "2025-05-02T08:33:15.458Z" }, + { url = "https://files.pythonhosted.org/packages/44/96/392abd49b094d30b91d9fbda6a69519e95802250b777841cf3bda8fe136c/charset_normalizer-3.4.2-cp313-cp313-win32.whl", hash = "sha256:aaeeb6a479c7667fbe1099af9617c83aaca22182d6cf8c53966491a0f1b7ffb7", size = 98064, upload-time = "2025-05-02T08:33:17.06Z" }, + { url = "https://files.pythonhosted.org/packages/e9/b0/0200da600134e001d91851ddc797809e2fe0ea72de90e09bec5a2fbdaccb/charset_normalizer-3.4.2-cp313-cp313-win_amd64.whl", hash = "sha256:aa6af9e7d59f9c12b33ae4e9450619cf2488e2bbe9b44030905877f0b2324980", size = 105641, upload-time = "2025-05-02T08:33:18.753Z" }, + { url = "https://files.pythonhosted.org/packages/20/94/c5790835a017658cbfabd07f3bfb549140c3ac458cfc196323996b10095a/charset_normalizer-3.4.2-py3-none-any.whl", hash = "sha256:7f56930ab0abd1c45cd15be65cc741c28b1c9a34876ce8c17a2fa107810c0af0", size = 52626, upload-time = "2025-05-02T08:34:40.053Z" }, +] + +[[package]] +name = "croniter" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "pytz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/2f/44d1ae153a0e27be56be43465e5cb39b9650c781e001e7864389deb25090/croniter-6.0.0.tar.gz", hash = "sha256:37c504b313956114a983ece2c2b07790b1f1094fe9d81cc94739214748255577", size = 64481, upload-time = "2024-12-17T17:17:47.32Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/4b/290b4c3efd6417a8b0c284896de19b1d5855e6dbdb97d2a35e68fa42de85/croniter-6.0.0-py2.py3-none-any.whl", hash = "sha256:2f878c3856f17896979b2a4379ba1f09c83e374931ea15cc835c5dd2eee9b368", size = 25468, upload-time = "2024-12-17T17:17:45.359Z" }, +] + +[[package]] +name = "edx" +version = "0.0.16" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "lxml" }, + { name = "requests" }, + { name = "urllib3" }, + { name = "zeep" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/35/87eeeeee86d8da9988faf99a77852dcb743c7b42408134dfa6a29bf0cefe/EDX-0.0.16.tar.gz", hash = "sha256:c48b98eb1e9c4343773631b436063c318c13948dda31b8f0691bcfa4052ae2c9", size = 20615, upload-time = "2024-01-03T08:07:16.679Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/4b/f7e4e3d0290cc7741877865fd8d9aaf1d18f8a1b1905477739f01043e69f/EDX-0.0.16-py3-none-any.whl", hash = "sha256:0a12bdf933c8f7ac63eb03e48b6a6442cda8ff9ae76b79eab57eb7ce36143d8a", size = 12521, upload-time = "2024-01-03T08:07:15.07Z" }, +] + +[[package]] +name = "elastic-transport" +version = "8.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/54/d498a766ac8fa475f931da85a154666cc81a70f8eb4a780bc8e4e934e9ac/elastic_transport-8.17.1.tar.gz", hash = "sha256:5edef32ac864dca8e2f0a613ef63491ee8d6b8cfb52881fa7313ba9290cac6d2", size = 73425, upload-time = "2025-03-13T07:28:30.776Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cf/cd/b71d5bc74cde7fc6fd9b2ff9389890f45d9762cbbbf81dc5e51fd7588c4a/elastic_transport-8.17.1-py3-none-any.whl", hash = "sha256:192718f498f1d10c5e9aa8b9cf32aed405e469a7f0e9d6a8923431dbb2c59fb8", size = 64969, upload-time = "2025-03-13T07:28:29.031Z" }, +] + +[[package]] +name = "elasticsearch" +version = "9.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "elastic-transport" }, + { name = "python-dateutil" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/39/58/0081e189ef83dd1f11cb600df842bffb8eaa05c097cb1672c5bd335b46c2/elasticsearch-9.0.1.tar.gz", hash = "sha256:76f9b519cfd15a860f615a94ba90dcb9543b721306bc5144ecc15f0b4d5d2781", size = 819519, upload-time = "2025-04-28T13:47:17.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/eb/03140387b1a378878a4228d0d796e80298ac54fc4fe6fe9e39087d142efe/elasticsearch-9.0.1-py3-none-any.whl", hash = "sha256:9fd110f9bb77310343709d9cccbd523a431a6e528d8441b01a26b97eef3238a3", size = 905510, upload-time = "2025-04-28T13:47:13.531Z" }, +] + +[[package]] +name = "emf" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "aniso8601" }, + { name = "croniter" }, + { name = "edx" }, + { name = "elasticsearch" }, + { name = "lxml" }, + { name = "minio" }, + { name = "ndjson" }, + { name = "opdm-api" }, + { name = "pandas" }, + { name = "pika" }, + { name = "pypowsybl" }, + { name = "pytz" }, + { name = "requests" }, + { name = "saxonche" }, + { name = "triplets" }, +] + +[package.metadata] +requires-dist = [ + { name = "aniso8601", specifier = ">=10.0.1" }, + { name = "croniter", specifier = ">=6.0.0" }, + { name = "edx", specifier = ">=0.0.16" }, + { name = "elasticsearch", specifier = ">=9.0.1" }, + { name = "lxml", specifier = ">=5.4.0" }, + { name = "minio", specifier = ">=7.2.15" }, + { name = "ndjson", specifier = ">=0.3.1" }, + { name = "opdm-api", specifier = ">=0.1.3" }, + { name = "pandas", specifier = "==2.2.2" }, + { name = "pika", specifier = ">=1.3.2" }, + { name = "pypowsybl", specifier = "==1.9.0" }, + { name = "pytz", specifier = ">=2025.2" }, + { name = "requests", specifier = ">=2.32.3" }, + { name = "saxonche", specifier = ">=12.7.0" }, + { name = "triplets", specifier = ">=0.0.10" }, +] + +[[package]] +name = "idna" +version = "3.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/70/7703c29685631f5a7590aa73f1f1d3fa9a380e654b86af429e0934a32f7d/idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9", size = 190490, upload-time = "2024-09-15T18:07:39.745Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload-time = "2024-09-15T18:07:37.964Z" }, +] + +[[package]] +name = "isodate" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, +] + +[[package]] +name = "lxml" +version = "5.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/3d/14e82fc7c8fb1b7761f7e748fd47e2ec8276d137b6acfe5a4bb73853e08f/lxml-5.4.0.tar.gz", hash = "sha256:d12832e1dbea4be280b22fd0ea7c9b87f0d8fc51ba06e92dc62d52f804f78ebd", size = 3679479, upload-time = "2025-04-23T01:50:29.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/2d/67693cc8a605a12e5975380d7ff83020dcc759351b5a066e1cced04f797b/lxml-5.4.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:98a3912194c079ef37e716ed228ae0dcb960992100461b704aea4e93af6b0bb9", size = 8083240, upload-time = "2025-04-23T01:45:18.566Z" }, + { url = "https://files.pythonhosted.org/packages/73/53/b5a05ab300a808b72e848efd152fe9c022c0181b0a70b8bca1199f1bed26/lxml-5.4.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:0ea0252b51d296a75f6118ed0d8696888e7403408ad42345d7dfd0d1e93309a7", size = 4387685, upload-time = "2025-04-23T01:45:21.387Z" }, + { url = "https://files.pythonhosted.org/packages/d8/cb/1a3879c5f512bdcd32995c301886fe082b2edd83c87d41b6d42d89b4ea4d/lxml-5.4.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b92b69441d1bd39f4940f9eadfa417a25862242ca2c396b406f9272ef09cdcaa", size = 4991164, upload-time = "2025-04-23T01:45:23.849Z" }, + { url = "https://files.pythonhosted.org/packages/f9/94/bbc66e42559f9d04857071e3b3d0c9abd88579367fd2588a4042f641f57e/lxml-5.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20e16c08254b9b6466526bc1828d9370ee6c0d60a4b64836bc3ac2917d1e16df", size = 4746206, upload-time = "2025-04-23T01:45:26.361Z" }, + { url = "https://files.pythonhosted.org/packages/66/95/34b0679bee435da2d7cae895731700e519a8dfcab499c21662ebe671603e/lxml-5.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7605c1c32c3d6e8c990dd28a0970a3cbbf1429d5b92279e37fda05fb0c92190e", size = 5342144, upload-time = "2025-04-23T01:45:28.939Z" }, + { url = "https://files.pythonhosted.org/packages/e0/5d/abfcc6ab2fa0be72b2ba938abdae1f7cad4c632f8d552683ea295d55adfb/lxml-5.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ecf4c4b83f1ab3d5a7ace10bafcb6f11df6156857a3c418244cef41ca9fa3e44", size = 4825124, upload-time = "2025-04-23T01:45:31.361Z" }, + { url = "https://files.pythonhosted.org/packages/5a/78/6bd33186c8863b36e084f294fc0a5e5eefe77af95f0663ef33809cc1c8aa/lxml-5.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0cef4feae82709eed352cd7e97ae062ef6ae9c7b5dbe3663f104cd2c0e8d94ba", size = 4876520, upload-time = "2025-04-23T01:45:34.191Z" }, + { url = "https://files.pythonhosted.org/packages/3b/74/4d7ad4839bd0fc64e3d12da74fc9a193febb0fae0ba6ebd5149d4c23176a/lxml-5.4.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:df53330a3bff250f10472ce96a9af28628ff1f4efc51ccba351a8820bca2a8ba", size = 4765016, upload-time = "2025-04-23T01:45:36.7Z" }, + { url = "https://files.pythonhosted.org/packages/24/0d/0a98ed1f2471911dadfc541003ac6dd6879fc87b15e1143743ca20f3e973/lxml-5.4.0-cp311-cp311-manylinux_2_28_ppc64le.whl", hash = "sha256:aefe1a7cb852fa61150fcb21a8c8fcea7b58c4cb11fbe59c97a0a4b31cae3c8c", size = 5362884, upload-time = "2025-04-23T01:45:39.291Z" }, + { url = "https://files.pythonhosted.org/packages/48/de/d4f7e4c39740a6610f0f6959052b547478107967362e8424e1163ec37ae8/lxml-5.4.0-cp311-cp311-manylinux_2_28_s390x.whl", hash = "sha256:ef5a7178fcc73b7d8c07229e89f8eb45b2908a9238eb90dcfc46571ccf0383b8", size = 4902690, upload-time = "2025-04-23T01:45:42.386Z" }, + { url = "https://files.pythonhosted.org/packages/07/8c/61763abd242af84f355ca4ef1ee096d3c1b7514819564cce70fd18c22e9a/lxml-5.4.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:d2ed1b3cb9ff1c10e6e8b00941bb2e5bb568b307bfc6b17dffbbe8be5eecba86", size = 4944418, upload-time = "2025-04-23T01:45:46.051Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c5/6d7e3b63e7e282619193961a570c0a4c8a57fe820f07ca3fe2f6bd86608a/lxml-5.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:72ac9762a9f8ce74c9eed4a4e74306f2f18613a6b71fa065495a67ac227b3056", size = 4827092, upload-time = "2025-04-23T01:45:48.943Z" }, + { url = "https://files.pythonhosted.org/packages/71/4a/e60a306df54680b103348545706a98a7514a42c8b4fbfdcaa608567bb065/lxml-5.4.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f5cb182f6396706dc6cc1896dd02b1c889d644c081b0cdec38747573db88a7d7", size = 5418231, upload-time = "2025-04-23T01:45:51.481Z" }, + { url = "https://files.pythonhosted.org/packages/27/f2/9754aacd6016c930875854f08ac4b192a47fe19565f776a64004aa167521/lxml-5.4.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:3a3178b4873df8ef9457a4875703488eb1622632a9cee6d76464b60e90adbfcd", size = 5261798, upload-time = "2025-04-23T01:45:54.146Z" }, + { url = "https://files.pythonhosted.org/packages/38/a2/0c49ec6941428b1bd4f280650d7b11a0f91ace9db7de32eb7aa23bcb39ff/lxml-5.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:e094ec83694b59d263802ed03a8384594fcce477ce484b0cbcd0008a211ca751", size = 4988195, upload-time = "2025-04-23T01:45:56.685Z" }, + { url = "https://files.pythonhosted.org/packages/7a/75/87a3963a08eafc46a86c1131c6e28a4de103ba30b5ae903114177352a3d7/lxml-5.4.0-cp311-cp311-win32.whl", hash = "sha256:4329422de653cdb2b72afa39b0aa04252fca9071550044904b2e7036d9d97fe4", size = 3474243, upload-time = "2025-04-23T01:45:58.863Z" }, + { url = "https://files.pythonhosted.org/packages/fa/f9/1f0964c4f6c2be861c50db380c554fb8befbea98c6404744ce243a3c87ef/lxml-5.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:fd3be6481ef54b8cfd0e1e953323b7aa9d9789b94842d0e5b142ef4bb7999539", size = 3815197, upload-time = "2025-04-23T01:46:01.096Z" }, + { url = "https://files.pythonhosted.org/packages/f8/4c/d101ace719ca6a4ec043eb516fcfcb1b396a9fccc4fcd9ef593df34ba0d5/lxml-5.4.0-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:b5aff6f3e818e6bdbbb38e5967520f174b18f539c2b9de867b1e7fde6f8d95a4", size = 8127392, upload-time = "2025-04-23T01:46:04.09Z" }, + { url = "https://files.pythonhosted.org/packages/11/84/beddae0cec4dd9ddf46abf156f0af451c13019a0fa25d7445b655ba5ccb7/lxml-5.4.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:942a5d73f739ad7c452bf739a62a0f83e2578afd6b8e5406308731f4ce78b16d", size = 4415103, upload-time = "2025-04-23T01:46:07.227Z" }, + { url = "https://files.pythonhosted.org/packages/d0/25/d0d93a4e763f0462cccd2b8a665bf1e4343dd788c76dcfefa289d46a38a9/lxml-5.4.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:460508a4b07364d6abf53acaa0a90b6d370fafde5693ef37602566613a9b0779", size = 5024224, upload-time = "2025-04-23T01:46:10.237Z" }, + { url = "https://files.pythonhosted.org/packages/31/ce/1df18fb8f7946e7f3388af378b1f34fcf253b94b9feedb2cec5969da8012/lxml-5.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:529024ab3a505fed78fe3cc5ddc079464e709f6c892733e3f5842007cec8ac6e", size = 4769913, upload-time = "2025-04-23T01:46:12.757Z" }, + { url = "https://files.pythonhosted.org/packages/4e/62/f4a6c60ae7c40d43657f552f3045df05118636be1165b906d3423790447f/lxml-5.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7ca56ebc2c474e8f3d5761debfd9283b8b18c76c4fc0967b74aeafba1f5647f9", size = 5290441, upload-time = "2025-04-23T01:46:16.037Z" }, + { url = "https://files.pythonhosted.org/packages/9e/aa/04f00009e1e3a77838c7fc948f161b5d2d5de1136b2b81c712a263829ea4/lxml-5.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a81e1196f0a5b4167a8dafe3a66aa67c4addac1b22dc47947abd5d5c7a3f24b5", size = 4820165, upload-time = "2025-04-23T01:46:19.137Z" }, + { url = "https://files.pythonhosted.org/packages/c9/1f/e0b2f61fa2404bf0f1fdf1898377e5bd1b74cc9b2cf2c6ba8509b8f27990/lxml-5.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00b8686694423ddae324cf614e1b9659c2edb754de617703c3d29ff568448df5", size = 4932580, upload-time = "2025-04-23T01:46:21.963Z" }, + { url = "https://files.pythonhosted.org/packages/24/a2/8263f351b4ffe0ed3e32ea7b7830f845c795349034f912f490180d88a877/lxml-5.4.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:c5681160758d3f6ac5b4fea370495c48aac0989d6a0f01bb9a72ad8ef5ab75c4", size = 4759493, upload-time = "2025-04-23T01:46:24.316Z" }, + { url = "https://files.pythonhosted.org/packages/05/00/41db052f279995c0e35c79d0f0fc9f8122d5b5e9630139c592a0b58c71b4/lxml-5.4.0-cp312-cp312-manylinux_2_28_ppc64le.whl", hash = "sha256:2dc191e60425ad70e75a68c9fd90ab284df64d9cd410ba8d2b641c0c45bc006e", size = 5324679, upload-time = "2025-04-23T01:46:27.097Z" }, + { url = "https://files.pythonhosted.org/packages/1d/be/ee99e6314cdef4587617d3b3b745f9356d9b7dd12a9663c5f3b5734b64ba/lxml-5.4.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:67f779374c6b9753ae0a0195a892a1c234ce8416e4448fe1e9f34746482070a7", size = 4890691, upload-time = "2025-04-23T01:46:30.009Z" }, + { url = "https://files.pythonhosted.org/packages/ad/36/239820114bf1d71f38f12208b9c58dec033cbcf80101cde006b9bde5cffd/lxml-5.4.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:79d5bfa9c1b455336f52343130b2067164040604e41f6dc4d8313867ed540079", size = 4955075, upload-time = "2025-04-23T01:46:32.33Z" }, + { url = "https://files.pythonhosted.org/packages/d4/e1/1b795cc0b174efc9e13dbd078a9ff79a58728a033142bc6d70a1ee8fc34d/lxml-5.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3d3c30ba1c9b48c68489dc1829a6eede9873f52edca1dda900066542528d6b20", size = 4838680, upload-time = "2025-04-23T01:46:34.852Z" }, + { url = "https://files.pythonhosted.org/packages/72/48/3c198455ca108cec5ae3662ae8acd7fd99476812fd712bb17f1b39a0b589/lxml-5.4.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:1af80c6316ae68aded77e91cd9d80648f7dd40406cef73df841aa3c36f6907c8", size = 5391253, upload-time = "2025-04-23T01:46:37.608Z" }, + { url = "https://files.pythonhosted.org/packages/d6/10/5bf51858971c51ec96cfc13e800a9951f3fd501686f4c18d7d84fe2d6352/lxml-5.4.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:4d885698f5019abe0de3d352caf9466d5de2baded00a06ef3f1216c1a58ae78f", size = 5261651, upload-time = "2025-04-23T01:46:40.183Z" }, + { url = "https://files.pythonhosted.org/packages/2b/11/06710dd809205377da380546f91d2ac94bad9ff735a72b64ec029f706c85/lxml-5.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:aea53d51859b6c64e7c51d522c03cc2c48b9b5d6172126854cc7f01aa11f52bc", size = 5024315, upload-time = "2025-04-23T01:46:43.333Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b0/15b6217834b5e3a59ebf7f53125e08e318030e8cc0d7310355e6edac98ef/lxml-5.4.0-cp312-cp312-win32.whl", hash = "sha256:d90b729fd2732df28130c064aac9bb8aff14ba20baa4aee7bd0795ff1187545f", size = 3486149, upload-time = "2025-04-23T01:46:45.684Z" }, + { url = "https://files.pythonhosted.org/packages/91/1e/05ddcb57ad2f3069101611bd5f5084157d90861a2ef460bf42f45cced944/lxml-5.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:1dc4ca99e89c335a7ed47d38964abcb36c5910790f9bd106f2a8fa2ee0b909d2", size = 3817095, upload-time = "2025-04-23T01:46:48.521Z" }, + { url = "https://files.pythonhosted.org/packages/87/cb/2ba1e9dd953415f58548506fa5549a7f373ae55e80c61c9041b7fd09a38a/lxml-5.4.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:773e27b62920199c6197130632c18fb7ead3257fce1ffb7d286912e56ddb79e0", size = 8110086, upload-time = "2025-04-23T01:46:52.218Z" }, + { url = "https://files.pythonhosted.org/packages/b5/3e/6602a4dca3ae344e8609914d6ab22e52ce42e3e1638c10967568c5c1450d/lxml-5.4.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ce9c671845de9699904b1e9df95acfe8dfc183f2310f163cdaa91a3535af95de", size = 4404613, upload-time = "2025-04-23T01:46:55.281Z" }, + { url = "https://files.pythonhosted.org/packages/4c/72/bf00988477d3bb452bef9436e45aeea82bb40cdfb4684b83c967c53909c7/lxml-5.4.0-cp313-cp313-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9454b8d8200ec99a224df8854786262b1bd6461f4280064c807303c642c05e76", size = 5012008, upload-time = "2025-04-23T01:46:57.817Z" }, + { url = "https://files.pythonhosted.org/packages/92/1f/93e42d93e9e7a44b2d3354c462cd784dbaaf350f7976b5d7c3f85d68d1b1/lxml-5.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cccd007d5c95279e529c146d095f1d39ac05139de26c098166c4beb9374b0f4d", size = 4760915, upload-time = "2025-04-23T01:47:00.745Z" }, + { url = "https://files.pythonhosted.org/packages/45/0b/363009390d0b461cf9976a499e83b68f792e4c32ecef092f3f9ef9c4ba54/lxml-5.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0fce1294a0497edb034cb416ad3e77ecc89b313cff7adbee5334e4dc0d11f422", size = 5283890, upload-time = "2025-04-23T01:47:04.702Z" }, + { url = "https://files.pythonhosted.org/packages/19/dc/6056c332f9378ab476c88e301e6549a0454dbee8f0ae16847414f0eccb74/lxml-5.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:24974f774f3a78ac12b95e3a20ef0931795ff04dbb16db81a90c37f589819551", size = 4812644, upload-time = "2025-04-23T01:47:07.833Z" }, + { url = "https://files.pythonhosted.org/packages/ee/8a/f8c66bbb23ecb9048a46a5ef9b495fd23f7543df642dabeebcb2eeb66592/lxml-5.4.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:497cab4d8254c2a90bf988f162ace2ddbfdd806fce3bda3f581b9d24c852e03c", size = 4921817, upload-time = "2025-04-23T01:47:10.317Z" }, + { url = "https://files.pythonhosted.org/packages/04/57/2e537083c3f381f83d05d9b176f0d838a9e8961f7ed8ddce3f0217179ce3/lxml-5.4.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:e794f698ae4c5084414efea0f5cc9f4ac562ec02d66e1484ff822ef97c2cadff", size = 4753916, upload-time = "2025-04-23T01:47:12.823Z" }, + { url = "https://files.pythonhosted.org/packages/d8/80/ea8c4072109a350848f1157ce83ccd9439601274035cd045ac31f47f3417/lxml-5.4.0-cp313-cp313-manylinux_2_28_ppc64le.whl", hash = "sha256:2c62891b1ea3094bb12097822b3d44b93fc6c325f2043c4d2736a8ff09e65f60", size = 5289274, upload-time = "2025-04-23T01:47:15.916Z" }, + { url = "https://files.pythonhosted.org/packages/b3/47/c4be287c48cdc304483457878a3f22999098b9a95f455e3c4bda7ec7fc72/lxml-5.4.0-cp313-cp313-manylinux_2_28_s390x.whl", hash = "sha256:142accb3e4d1edae4b392bd165a9abdee8a3c432a2cca193df995bc3886249c8", size = 4874757, upload-time = "2025-04-23T01:47:19.793Z" }, + { url = "https://files.pythonhosted.org/packages/2f/04/6ef935dc74e729932e39478e44d8cfe6a83550552eaa072b7c05f6f22488/lxml-5.4.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:1a42b3a19346e5601d1b8296ff6ef3d76038058f311902edd574461e9c036982", size = 4947028, upload-time = "2025-04-23T01:47:22.401Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f9/c33fc8daa373ef8a7daddb53175289024512b6619bc9de36d77dca3df44b/lxml-5.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:4291d3c409a17febf817259cb37bc62cb7eb398bcc95c1356947e2871911ae61", size = 4834487, upload-time = "2025-04-23T01:47:25.513Z" }, + { url = "https://files.pythonhosted.org/packages/8d/30/fc92bb595bcb878311e01b418b57d13900f84c2b94f6eca9e5073ea756e6/lxml-5.4.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:4f5322cf38fe0e21c2d73901abf68e6329dc02a4994e483adbcf92b568a09a54", size = 5381688, upload-time = "2025-04-23T01:47:28.454Z" }, + { url = "https://files.pythonhosted.org/packages/43/d1/3ba7bd978ce28bba8e3da2c2e9d5ae3f8f521ad3f0ca6ea4788d086ba00d/lxml-5.4.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:0be91891bdb06ebe65122aa6bf3fc94489960cf7e03033c6f83a90863b23c58b", size = 5242043, upload-time = "2025-04-23T01:47:31.208Z" }, + { url = "https://files.pythonhosted.org/packages/ee/cd/95fa2201041a610c4d08ddaf31d43b98ecc4b1d74b1e7245b1abdab443cb/lxml-5.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:15a665ad90054a3d4f397bc40f73948d48e36e4c09f9bcffc7d90c87410e478a", size = 5021569, upload-time = "2025-04-23T01:47:33.805Z" }, + { url = "https://files.pythonhosted.org/packages/2d/a6/31da006fead660b9512d08d23d31e93ad3477dd47cc42e3285f143443176/lxml-5.4.0-cp313-cp313-win32.whl", hash = "sha256:d5663bc1b471c79f5c833cffbc9b87d7bf13f87e055a5c86c363ccd2348d7e82", size = 3485270, upload-time = "2025-04-23T01:47:36.133Z" }, + { url = "https://files.pythonhosted.org/packages/fc/14/c115516c62a7d2499781d2d3d7215218c0731b2c940753bf9f9b7b73924d/lxml-5.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:bcb7a1096b4b6b24ce1ac24d4942ad98f983cd3810f9711bcd0293f43a9d8b9f", size = 3814606, upload-time = "2025-04-23T01:47:39.028Z" }, +] + +[[package]] +name = "minio" +version = "7.2.15" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "argon2-cffi" }, + { name = "certifi" }, + { name = "pycryptodome" }, + { name = "typing-extensions" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/68/86a1cef80396e6a35a6fc4fafee5d28578c1a137bddd3ca2aa86f9b26a22/minio-7.2.15.tar.gz", hash = "sha256:5247df5d4dca7bfa4c9b20093acd5ad43e82d8710ceb059d79c6eea970f49f79", size = 138040, upload-time = "2025-01-19T08:57:26.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/6f/3690028e846fe432bfa5ba724a0dc37ec9c914965b7733e19d8ca2c4c48d/minio-7.2.15-py3-none-any.whl", hash = "sha256:c06ef7a43e5d67107067f77b6c07ebdd68733e5aa7eed03076472410ca19d876", size = 95075, upload-time = "2025-01-19T08:57:24.169Z" }, +] + +[[package]] +name = "ndjson" +version = "0.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/d5/209b6ca94566f9c94c0ec41cee1681c0a3b92a306a84a9b0fcd662088dc3/ndjson-0.3.1.tar.gz", hash = "sha256:bf9746cb6bb1cb53d172cda7f154c07c786d665ff28341e4e689b796b229e5d6", size = 6448, upload-time = "2020-02-25T05:01:07.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/c9/04ba0056011ba96a58163ebfd666d8385300bd12da1afe661a5a147758d7/ndjson-0.3.1-py2.py3-none-any.whl", hash = "sha256:839c22275e6baa3040077b83c005ac24199b94973309a8a1809be962c753a410", size = 5305, upload-time = "2020-02-25T05:01:06.39Z" }, +] + +[[package]] +name = "networkx" +version = "3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, +] + +[[package]] +name = "numpy" +version = "2.2.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/76/21/7d2a95e4bba9dc13d043ee156a356c0a8f0c6309dff6b21b4d71a073b8a8/numpy-2.2.6.tar.gz", hash = "sha256:e29554e2bef54a90aa5cc07da6ce955accb83f21ab5de01a62c8478897b264fd", size = 20276440, upload-time = "2025-05-17T22:38:04.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/a8/4f83e2aa666a9fbf56d6118faaaf5f1974d456b1823fda0a176eff722839/numpy-2.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f9f1adb22318e121c5c69a09142811a201ef17ab257a1e66ca3025065b7f53ae", size = 21176963, upload-time = "2025-05-17T21:31:19.36Z" }, + { url = "https://files.pythonhosted.org/packages/b3/2b/64e1affc7972decb74c9e29e5649fac940514910960ba25cd9af4488b66c/numpy-2.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c820a93b0255bc360f53eca31a0e676fd1101f673dda8da93454a12e23fc5f7a", size = 14406743, upload-time = "2025-05-17T21:31:41.087Z" }, + { url = "https://files.pythonhosted.org/packages/4a/9f/0121e375000b5e50ffdd8b25bf78d8e1a5aa4cca3f185d41265198c7b834/numpy-2.2.6-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:3d70692235e759f260c3d837193090014aebdf026dfd167834bcba43e30c2a42", size = 5352616, upload-time = "2025-05-17T21:31:50.072Z" }, + { url = "https://files.pythonhosted.org/packages/31/0d/b48c405c91693635fbe2dcd7bc84a33a602add5f63286e024d3b6741411c/numpy-2.2.6-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:481b49095335f8eed42e39e8041327c05b0f6f4780488f61286ed3c01368d491", size = 6889579, upload-time = "2025-05-17T21:32:01.712Z" }, + { url = "https://files.pythonhosted.org/packages/52/b8/7f0554d49b565d0171eab6e99001846882000883998e7b7d9f0d98b1f934/numpy-2.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b64d8d4d17135e00c8e346e0a738deb17e754230d7e0810ac5012750bbd85a5a", size = 14312005, upload-time = "2025-05-17T21:32:23.332Z" }, + { url = "https://files.pythonhosted.org/packages/b3/dd/2238b898e51bd6d389b7389ffb20d7f4c10066d80351187ec8e303a5a475/numpy-2.2.6-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba10f8411898fc418a521833e014a77d3ca01c15b0c6cdcce6a0d2897e6dbbdf", size = 16821570, upload-time = "2025-05-17T21:32:47.991Z" }, + { url = "https://files.pythonhosted.org/packages/83/6c/44d0325722cf644f191042bf47eedad61c1e6df2432ed65cbe28509d404e/numpy-2.2.6-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:bd48227a919f1bafbdda0583705e547892342c26fb127219d60a5c36882609d1", size = 15818548, upload-time = "2025-05-17T21:33:11.728Z" }, + { url = "https://files.pythonhosted.org/packages/ae/9d/81e8216030ce66be25279098789b665d49ff19eef08bfa8cb96d4957f422/numpy-2.2.6-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:9551a499bf125c1d4f9e250377c1ee2eddd02e01eac6644c080162c0c51778ab", size = 18620521, upload-time = "2025-05-17T21:33:39.139Z" }, + { url = "https://files.pythonhosted.org/packages/6a/fd/e19617b9530b031db51b0926eed5345ce8ddc669bb3bc0044b23e275ebe8/numpy-2.2.6-cp311-cp311-win32.whl", hash = "sha256:0678000bb9ac1475cd454c6b8c799206af8107e310843532b04d49649c717a47", size = 6525866, upload-time = "2025-05-17T21:33:50.273Z" }, + { url = "https://files.pythonhosted.org/packages/31/0a/f354fb7176b81747d870f7991dc763e157a934c717b67b58456bc63da3df/numpy-2.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:e8213002e427c69c45a52bbd94163084025f533a55a59d6f9c5b820774ef3303", size = 12907455, upload-time = "2025-05-17T21:34:09.135Z" }, + { url = "https://files.pythonhosted.org/packages/82/5d/c00588b6cf18e1da539b45d3598d3557084990dcc4331960c15ee776ee41/numpy-2.2.6-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:41c5a21f4a04fa86436124d388f6ed60a9343a6f767fced1a8a71c3fbca038ff", size = 20875348, upload-time = "2025-05-17T21:34:39.648Z" }, + { url = "https://files.pythonhosted.org/packages/66/ee/560deadcdde6c2f90200450d5938f63a34b37e27ebff162810f716f6a230/numpy-2.2.6-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:de749064336d37e340f640b05f24e9e3dd678c57318c7289d222a8a2f543e90c", size = 14119362, upload-time = "2025-05-17T21:35:01.241Z" }, + { url = "https://files.pythonhosted.org/packages/3c/65/4baa99f1c53b30adf0acd9a5519078871ddde8d2339dc5a7fde80d9d87da/numpy-2.2.6-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:894b3a42502226a1cac872f840030665f33326fc3dac8e57c607905773cdcde3", size = 5084103, upload-time = "2025-05-17T21:35:10.622Z" }, + { url = "https://files.pythonhosted.org/packages/cc/89/e5a34c071a0570cc40c9a54eb472d113eea6d002e9ae12bb3a8407fb912e/numpy-2.2.6-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:71594f7c51a18e728451bb50cc60a3ce4e6538822731b2933209a1f3614e9282", size = 6625382, upload-time = "2025-05-17T21:35:21.414Z" }, + { url = "https://files.pythonhosted.org/packages/f8/35/8c80729f1ff76b3921d5c9487c7ac3de9b2a103b1cd05e905b3090513510/numpy-2.2.6-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f2618db89be1b4e05f7a1a847a9c1c0abd63e63a1607d892dd54668dd92faf87", size = 14018462, upload-time = "2025-05-17T21:35:42.174Z" }, + { url = "https://files.pythonhosted.org/packages/8c/3d/1e1db36cfd41f895d266b103df00ca5b3cbe965184df824dec5c08c6b803/numpy-2.2.6-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fd83c01228a688733f1ded5201c678f0c53ecc1006ffbc404db9f7a899ac6249", size = 16527618, upload-time = "2025-05-17T21:36:06.711Z" }, + { url = "https://files.pythonhosted.org/packages/61/c6/03ed30992602c85aa3cd95b9070a514f8b3c33e31124694438d88809ae36/numpy-2.2.6-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:37c0ca431f82cd5fa716eca9506aefcabc247fb27ba69c5062a6d3ade8cf8f49", size = 15505511, upload-time = "2025-05-17T21:36:29.965Z" }, + { url = "https://files.pythonhosted.org/packages/b7/25/5761d832a81df431e260719ec45de696414266613c9ee268394dd5ad8236/numpy-2.2.6-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:fe27749d33bb772c80dcd84ae7e8df2adc920ae8297400dabec45f0dedb3f6de", size = 18313783, upload-time = "2025-05-17T21:36:56.883Z" }, + { url = "https://files.pythonhosted.org/packages/57/0a/72d5a3527c5ebffcd47bde9162c39fae1f90138c961e5296491ce778e682/numpy-2.2.6-cp312-cp312-win32.whl", hash = "sha256:4eeaae00d789f66c7a25ac5f34b71a7035bb474e679f410e5e1a94deb24cf2d4", size = 6246506, upload-time = "2025-05-17T21:37:07.368Z" }, + { url = "https://files.pythonhosted.org/packages/36/fa/8c9210162ca1b88529ab76b41ba02d433fd54fecaf6feb70ef9f124683f1/numpy-2.2.6-cp312-cp312-win_amd64.whl", hash = "sha256:c1f9540be57940698ed329904db803cf7a402f3fc200bfe599334c9bd84a40b2", size = 12614190, upload-time = "2025-05-17T21:37:26.213Z" }, + { url = "https://files.pythonhosted.org/packages/f9/5c/6657823f4f594f72b5471f1db1ab12e26e890bb2e41897522d134d2a3e81/numpy-2.2.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0811bb762109d9708cca4d0b13c4f67146e3c3b7cf8d34018c722adb2d957c84", size = 20867828, upload-time = "2025-05-17T21:37:56.699Z" }, + { url = "https://files.pythonhosted.org/packages/dc/9e/14520dc3dadf3c803473bd07e9b2bd1b69bc583cb2497b47000fed2fa92f/numpy-2.2.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:287cc3162b6f01463ccd86be154f284d0893d2b3ed7292439ea97eafa8170e0b", size = 14143006, upload-time = "2025-05-17T21:38:18.291Z" }, + { url = "https://files.pythonhosted.org/packages/4f/06/7e96c57d90bebdce9918412087fc22ca9851cceaf5567a45c1f404480e9e/numpy-2.2.6-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:f1372f041402e37e5e633e586f62aa53de2eac8d98cbfb822806ce4bbefcb74d", size = 5076765, upload-time = "2025-05-17T21:38:27.319Z" }, + { url = "https://files.pythonhosted.org/packages/73/ed/63d920c23b4289fdac96ddbdd6132e9427790977d5457cd132f18e76eae0/numpy-2.2.6-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:55a4d33fa519660d69614a9fad433be87e5252f4b03850642f88993f7b2ca566", size = 6617736, upload-time = "2025-05-17T21:38:38.141Z" }, + { url = "https://files.pythonhosted.org/packages/85/c5/e19c8f99d83fd377ec8c7e0cf627a8049746da54afc24ef0a0cb73d5dfb5/numpy-2.2.6-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f92729c95468a2f4f15e9bb94c432a9229d0d50de67304399627a943201baa2f", size = 14010719, upload-time = "2025-05-17T21:38:58.433Z" }, + { url = "https://files.pythonhosted.org/packages/19/49/4df9123aafa7b539317bf6d342cb6d227e49f7a35b99c287a6109b13dd93/numpy-2.2.6-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bc23a79bfabc5d056d106f9befb8d50c31ced2fbc70eedb8155aec74a45798f", size = 16526072, upload-time = "2025-05-17T21:39:22.638Z" }, + { url = "https://files.pythonhosted.org/packages/b2/6c/04b5f47f4f32f7c2b0e7260442a8cbcf8168b0e1a41ff1495da42f42a14f/numpy-2.2.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:e3143e4451880bed956e706a3220b4e5cf6172ef05fcc397f6f36a550b1dd868", size = 15503213, upload-time = "2025-05-17T21:39:45.865Z" }, + { url = "https://files.pythonhosted.org/packages/17/0a/5cd92e352c1307640d5b6fec1b2ffb06cd0dabe7d7b8227f97933d378422/numpy-2.2.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b4f13750ce79751586ae2eb824ba7e1e8dba64784086c98cdbbcc6a42112ce0d", size = 18316632, upload-time = "2025-05-17T21:40:13.331Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3b/5cba2b1d88760ef86596ad0f3d484b1cbff7c115ae2429678465057c5155/numpy-2.2.6-cp313-cp313-win32.whl", hash = "sha256:5beb72339d9d4fa36522fc63802f469b13cdbe4fdab4a288f0c441b74272ebfd", size = 6244532, upload-time = "2025-05-17T21:43:46.099Z" }, + { url = "https://files.pythonhosted.org/packages/cb/3b/d58c12eafcb298d4e6d0d40216866ab15f59e55d148a5658bb3132311fcf/numpy-2.2.6-cp313-cp313-win_amd64.whl", hash = "sha256:b0544343a702fa80c95ad5d3d608ea3599dd54d4632df855e4c8d24eb6ecfa1c", size = 12610885, upload-time = "2025-05-17T21:44:05.145Z" }, + { url = "https://files.pythonhosted.org/packages/6b/9e/4bf918b818e516322db999ac25d00c75788ddfd2d2ade4fa66f1f38097e1/numpy-2.2.6-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0bca768cd85ae743b2affdc762d617eddf3bcf8724435498a1e80132d04879e6", size = 20963467, upload-time = "2025-05-17T21:40:44Z" }, + { url = "https://files.pythonhosted.org/packages/61/66/d2de6b291507517ff2e438e13ff7b1e2cdbdb7cb40b3ed475377aece69f9/numpy-2.2.6-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:fc0c5673685c508a142ca65209b4e79ed6740a4ed6b2267dbba90f34b0b3cfda", size = 14225144, upload-time = "2025-05-17T21:41:05.695Z" }, + { url = "https://files.pythonhosted.org/packages/e4/25/480387655407ead912e28ba3a820bc69af9adf13bcbe40b299d454ec011f/numpy-2.2.6-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:5bd4fc3ac8926b3819797a7c0e2631eb889b4118a9898c84f585a54d475b7e40", size = 5200217, upload-time = "2025-05-17T21:41:15.903Z" }, + { url = "https://files.pythonhosted.org/packages/aa/4a/6e313b5108f53dcbf3aca0c0f3e9c92f4c10ce57a0a721851f9785872895/numpy-2.2.6-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:fee4236c876c4e8369388054d02d0e9bb84821feb1a64dd59e137e6511a551f8", size = 6712014, upload-time = "2025-05-17T21:41:27.321Z" }, + { url = "https://files.pythonhosted.org/packages/b7/30/172c2d5c4be71fdf476e9de553443cf8e25feddbe185e0bd88b096915bcc/numpy-2.2.6-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e1dda9c7e08dc141e0247a5b8f49cf05984955246a327d4c48bda16821947b2f", size = 14077935, upload-time = "2025-05-17T21:41:49.738Z" }, + { url = "https://files.pythonhosted.org/packages/12/fb/9e743f8d4e4d3c710902cf87af3512082ae3d43b945d5d16563f26ec251d/numpy-2.2.6-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f447e6acb680fd307f40d3da4852208af94afdfab89cf850986c3ca00562f4fa", size = 16600122, upload-time = "2025-05-17T21:42:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/12/75/ee20da0e58d3a66f204f38916757e01e33a9737d0b22373b3eb5a27358f9/numpy-2.2.6-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:389d771b1623ec92636b0786bc4ae56abafad4a4c513d36a55dce14bd9ce8571", size = 15586143, upload-time = "2025-05-17T21:42:37.464Z" }, + { url = "https://files.pythonhosted.org/packages/76/95/bef5b37f29fc5e739947e9ce5179ad402875633308504a52d188302319c8/numpy-2.2.6-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8e9ace4a37db23421249ed236fdcdd457d671e25146786dfc96835cd951aa7c1", size = 18385260, upload-time = "2025-05-17T21:43:05.189Z" }, + { url = "https://files.pythonhosted.org/packages/09/04/f2f83279d287407cf36a7a8053a5abe7be3622a4363337338f2585e4afda/numpy-2.2.6-cp313-cp313t-win32.whl", hash = "sha256:038613e9fb8c72b0a41f025a7e4c3f0b7a1b5d768ece4796b674c8f3fe13efff", size = 6377225, upload-time = "2025-05-17T21:43:16.254Z" }, + { url = "https://files.pythonhosted.org/packages/67/0e/35082d13c09c02c011cf21570543d202ad929d961c02a147493cb0c2bdf5/numpy-2.2.6-cp313-cp313t-win_amd64.whl", hash = "sha256:6031dd6dfecc0cf9f668681a37648373bddd6421fff6c66ec1624eed0180ee06", size = 12771374, upload-time = "2025-05-17T21:43:35.479Z" }, +] + +[[package]] +name = "opdm-api" +version = "0.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aniso8601" }, + { name = "lxml" }, + { name = "requests" }, + { name = "urllib3" }, + { name = "xmltodict" }, + { name = "zeep" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/2b/eef02b44c13013e66873094cf0da98731f34448c6628907d65669407cb9e/opdm-api-0.1.3.tar.gz", hash = "sha256:e5e1884d848190232ec62b7467e80451286a0c3c845faa87fbbf68837cc12c35", size = 26528, upload-time = "2025-05-21T11:09:26.081Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/95/15a9378972a1ac1a4ee2a7fee0fbb59ed443e72323c0cb7616ed539f52f2/opdm_api-0.1.3-py3-none-any.whl", hash = "sha256:91913afc4a74e5fdeea5ba4c3442382342e0031b6384dbb05568691ff9495ef2", size = 11497, upload-time = "2025-05-21T11:09:22.315Z" }, +] + +[[package]] +name = "pandas" +version = "2.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/88/d9/ecf715f34c73ccb1d8ceb82fc01cd1028a65a5f6dbc57bfa6ea155119058/pandas-2.2.2.tar.gz", hash = "sha256:9e79019aba43cb4fda9e4d983f8e88ca0373adbb697ae9c6c43093218de28b54", size = 4398391, upload-time = "2024-04-10T19:45:48.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/70/61704497903d43043e288017cb2b82155c0d41e15f5c17807920877b45c2/pandas-2.2.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:696039430f7a562b74fa45f540aca068ea85fa34c244d0deee539cb6d70aa288", size = 12574808, upload-time = "2024-04-10T19:44:35.516Z" }, + { url = "https://files.pythonhosted.org/packages/16/c6/75231fd47afd6b3f89011e7077f1a3958441264aca7ae9ff596e3276a5d0/pandas-2.2.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e90497254aacacbc4ea6ae5e7a8cd75629d6ad2b30025a4a8b09aa4faf55151", size = 11304876, upload-time = "2024-04-10T19:44:39.37Z" }, + { url = "https://files.pythonhosted.org/packages/97/2d/7b54f80b93379ff94afb3bd9b0cd1d17b48183a0d6f98045bc01ce1e06a7/pandas-2.2.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58b84b91b0b9f4bafac2a0ac55002280c094dfc6402402332c0913a59654ab2b", size = 15602548, upload-time = "2024-04-10T19:44:42.902Z" }, + { url = "https://files.pythonhosted.org/packages/fc/a5/4d82be566f069d7a9a702dcdf6f9106df0e0b042e738043c0cc7ddd7e3f6/pandas-2.2.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6d2123dc9ad6a814bcdea0f099885276b31b24f7edf40f6cdbc0912672e22eee", size = 13031332, upload-time = "2024-04-10T19:44:46.98Z" }, + { url = "https://files.pythonhosted.org/packages/92/a2/b79c48f530673567805e607712b29814b47dcaf0d167e87145eb4b0118c6/pandas-2.2.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:2925720037f06e89af896c70bca73459d7e6a4be96f9de79e2d440bd499fe0db", size = 16286054, upload-time = "2024-04-10T19:44:50.51Z" }, + { url = "https://files.pythonhosted.org/packages/40/c7/47e94907f1d8fdb4868d61bd6c93d57b3784a964d52691b77ebfdb062842/pandas-2.2.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0cace394b6ea70c01ca1595f839cf193df35d1575986e484ad35c4aeae7266c1", size = 13879507, upload-time = "2024-04-10T19:44:54.412Z" }, + { url = "https://files.pythonhosted.org/packages/ab/63/966db1321a0ad55df1d1fe51505d2cdae191b84c907974873817b0a6e849/pandas-2.2.2-cp311-cp311-win_amd64.whl", hash = "sha256:873d13d177501a28b2756375d59816c365e42ed8417b41665f346289adc68d24", size = 11634249, upload-time = "2024-04-10T19:44:58.183Z" }, + { url = "https://files.pythonhosted.org/packages/dd/49/de869130028fb8d90e25da3b7d8fb13e40f5afa4c4af1781583eb1ff3839/pandas-2.2.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:9dfde2a0ddef507a631dc9dc4af6a9489d5e2e740e226ad426a05cabfbd7c8ef", size = 12500886, upload-time = "2024-04-10T19:45:01.808Z" }, + { url = "https://files.pythonhosted.org/packages/db/7c/9a60add21b96140e22465d9adf09832feade45235cd22f4cb1668a25e443/pandas-2.2.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e9b79011ff7a0f4b1d6da6a61aa1aa604fb312d6647de5bad20013682d1429ce", size = 11340320, upload-time = "2024-04-11T18:36:14.398Z" }, + { url = "https://files.pythonhosted.org/packages/b0/85/f95b5f322e1ae13b7ed7e97bd999160fa003424711ab4dc8344b8772c270/pandas-2.2.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1cb51fe389360f3b5a4d57dbd2848a5f033350336ca3b340d1c53a1fad33bcad", size = 15204346, upload-time = "2024-04-10T19:45:05.903Z" }, + { url = "https://files.pythonhosted.org/packages/40/10/79e52ef01dfeb1c1ca47a109a01a248754ebe990e159a844ece12914de83/pandas-2.2.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eee3a87076c0756de40b05c5e9a6069c035ba43e8dd71c379e68cab2c20f16ad", size = 12733396, upload-time = "2024-04-10T19:45:09.282Z" }, + { url = "https://files.pythonhosted.org/packages/35/9d/208febf8c4eb5c1d9ea3314d52d8bd415fd0ef0dd66bb24cc5bdbc8fa71a/pandas-2.2.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e374f59e440d4ab45ca2fffde54b81ac3834cf5ae2cdfa69c90bc03bde04d76", size = 15858913, upload-time = "2024-04-10T19:45:12.514Z" }, + { url = "https://files.pythonhosted.org/packages/99/d1/2d9bd05def7a9e08a92ec929b5a4c8d5556ec76fae22b0fa486cbf33ea63/pandas-2.2.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:43498c0bdb43d55cb162cdc8c06fac328ccb5d2eabe3cadeb3529ae6f0517c32", size = 13417786, upload-time = "2024-04-10T19:45:16.275Z" }, + { url = "https://files.pythonhosted.org/packages/22/a5/a0b255295406ed54269814bc93723cfd1a0da63fb9aaf99e1364f07923e5/pandas-2.2.2-cp312-cp312-win_amd64.whl", hash = "sha256:d187d355ecec3629624fccb01d104da7d7f391db0311145817525281e2804d23", size = 11498828, upload-time = "2024-04-10T19:45:19.85Z" }, +] + +[[package]] +name = "pika" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/db/db/d4102f356af18f316c67f2cead8ece307f731dd63140e2c71f170ddacf9b/pika-1.3.2.tar.gz", hash = "sha256:b2a327ddddf8570b4965b3576ac77091b850262d34ce8c1d8cb4e4146aa4145f", size = 145029, upload-time = "2023-05-05T14:25:43.368Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/f3/f412836ec714d36f0f4ab581b84c491e3f42c6b5b97a6c6ed1817f3c16d0/pika-1.3.2-py3-none-any.whl", hash = "sha256:0779a7c1fafd805672796085560d290213a465e4f6f76a6fb19e378d8041a14f", size = 155415, upload-time = "2023-05-05T14:25:41.484Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.3.8" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/8b/3c73abc9c759ecd3f1f7ceff6685840859e8070c4d947c93fae71f6a0bf2/platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc", size = 21362, upload-time = "2025-05-07T22:47:42.121Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fe/39/979e8e21520d4e47a0bbe349e2713c0aac6f3d853d0e5b34d76206c439aa/platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4", size = 18567, upload-time = "2025-05-07T22:47:40.376Z" }, +] + +[[package]] +name = "prettytable" +version = "3.16.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/99/b1/85e18ac92afd08c533603e3393977b6bc1443043115a47bb094f3b98f94f/prettytable-3.16.0.tar.gz", hash = "sha256:3c64b31719d961bf69c9a7e03d0c1e477320906a98da63952bc6698d6164ff57", size = 66276, upload-time = "2025-03-24T19:39:04.008Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/c7/5613524e606ea1688b3bdbf48aa64bafb6d0a4ac3750274c43b6158a390f/prettytable-3.16.0-py3-none-any.whl", hash = "sha256:b5eccfabb82222f5aa46b798ff02a8452cf530a352c31bddfa29be41242863aa", size = 33863, upload-time = "2025-03-24T19:39:02.359Z" }, +] + +[[package]] +name = "pycparser" +version = "2.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/b2/31537cf4b1ca988837256c910a668b553fceb8f069bedc4b1c826024b52c/pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6", size = 172736, upload-time = "2024-03-30T13:22:22.564Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/13/a3/a812df4e2dd5696d1f351d58b8fe16a405b234ad2886a0dab9183fb78109/pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc", size = 117552, upload-time = "2024-03-30T13:22:20.476Z" }, +] + +[[package]] +name = "pycryptodome" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" }, + { url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" }, + { url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" }, + { url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" }, + { url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" }, + { url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" }, + { url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" }, + { url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" }, + { url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" }, + { url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" }, + { url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" }, + { url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" }, + { url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" }, + { url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" }, + { url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" }, + { url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" }, + { url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" }, + { url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" }, + { url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" }, + { url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" }, + { url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" }, +] + +[[package]] +name = "pypowsybl" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "networkx" }, + { name = "pandas" }, + { name = "prettytable" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/fc/3ed067e2bb12f11deb06eff55cc076d8cefbc50d3dd32fd84753c40291ce/pypowsybl-1.9.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4f06d9007395b92802a1f7ce891de915265bcff8aa6ac8baca4a93436e091ca3", size = 62696454, upload-time = "2025-01-14T14:02:12.249Z" }, + { url = "https://files.pythonhosted.org/packages/2c/61/c50010a634daf2763223faec0f8813c9ae4e66fb142649ffb2210b33d9fc/pypowsybl-1.9.0-cp311-cp311-macosx_11_0_x86_64.whl", hash = "sha256:06a9ac540861f1bf271838c6151a61d35886b3f89ab23a29dc88d1f42392d6b6", size = 65553156, upload-time = "2025-01-14T14:03:06.163Z" }, + { url = "https://files.pythonhosted.org/packages/af/e0/fe0433cca394904263980af97c3718e1e01a2f6df7a3f358441dc534d09d/pypowsybl-1.9.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ce052c2b2bcf8fd044cab613bd220806d6d74cce3b4cf56842f6d0aa35115579", size = 78710140, upload-time = "2025-01-14T14:04:01.942Z" }, + { url = "https://files.pythonhosted.org/packages/b5/fb/4fc0efac3ec83c9816c1d3c71f15f51b4997878470378b59755cc45371cb/pypowsybl-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:1015e4d0acc51634fe476a31a05e4b5827afe6cef2a5aa1a9b9ff97bba90c1a1", size = 61193440, upload-time = "2025-01-14T14:04:56.03Z" }, + { url = "https://files.pythonhosted.org/packages/53/ee/1e6bbf29a252f7da2382fb1d4b42b3a6c2dd51c3b263472185fdf9d46b83/pypowsybl-1.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:804b8a46989f303f5ef8851c2ba798fc7cf7b157b4ff3a9eb6a4e74bd117a5fa", size = 62708437, upload-time = "2025-01-14T14:02:24.731Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/1ef99b6d8ffed047f17a449094e04483bf8c3411b54ec5d3d9488fcf0056/pypowsybl-1.9.0-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:d76af6ee9ad031889e3356758c2d9e313c6cd5b625d6e72389a6c976e644f75a", size = 65555446, upload-time = "2025-01-14T14:03:19.976Z" }, + { url = "https://files.pythonhosted.org/packages/e3/41/b14fd11d372363db6e3f640ccc23949bf579e15da1ed0575dfffdc330694/pypowsybl-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9254f48c09e19fdef00f776f09c636976ee7373be7a2ef7209709993dadca40f", size = 78699091, upload-time = "2025-01-14T14:04:13.283Z" }, + { url = "https://files.pythonhosted.org/packages/c6/33/24f023ced3db9695637e4e14a2bda3a3875f423a376f9e0e74f0bb224664/pypowsybl-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:8728f39fdc94cc22bae4e4189f0875cf0e2d7b8711c012522322a784a6d6a2b1", size = 61202792, upload-time = "2025-01-14T14:05:05.966Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytz" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f8/bf/abbd3cdfb8fbc7fb3d4d38d320f2441b1e7cbe29be4f23797b4a2b5d8aac/pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3", size = 320884, upload-time = "2025-03-25T02:25:00.538Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/c4/34e93fe5f5429d7570ec1fa436f1986fb1f00c3e0f43a589fe2bbcd22c3f/pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00", size = 509225, upload-time = "2025-03-25T02:24:58.468Z" }, +] + +[[package]] +name = "requests" +version = "2.32.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/63/70/2bf7780ad2d390a8d301ad0b550f1581eadbd9a20f896afe06353c2a2913/requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", size = 131218, upload-time = "2024-05-29T15:37:49.536Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928, upload-time = "2024-05-29T15:37:47.027Z" }, +] + +[[package]] +name = "requests-file" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/72/97/bf44e6c6bd8ddbb99943baf7ba8b1a8485bcd2fe0e55e5708d7fee4ff1ae/requests_file-2.1.0.tar.gz", hash = "sha256:0f549a3f3b0699415ac04d167e9cb39bccfb730cb832b4d20be3d9867356e658", size = 6891, upload-time = "2024-05-21T16:28:00.24Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/25/dd878a121fcfdf38f52850f11c512e13ec87c2ea72385933818e5b6c15ce/requests_file-2.1.0-py2.py3-none-any.whl", hash = "sha256:cf270de5a4c5874e84599fc5778303d496c10ae5e870bfa378818f35d21bda5c", size = 4244, upload-time = "2024-05-21T16:27:57.733Z" }, +] + +[[package]] +name = "requests-toolbelt" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f3/61/d7545dafb7ac2230c70d38d31cbfe4cc64f7144dc41f6e4e4b78ecd9f5bb/requests-toolbelt-1.0.0.tar.gz", hash = "sha256:7681a0a3d047012b5bdc0ee37d7f8f07ebe76ab08caeccfc3921ce23c88d5bc6", size = 206888, upload-time = "2023-05-01T04:11:33.229Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/51/d4db610ef29373b879047326cbf6fa98b6c1969d6f6dc423279de2b1be2c/requests_toolbelt-1.0.0-py2.py3-none-any.whl", hash = "sha256:cccfdd665f0a24fcf4726e690f65639d272bb0637b9b92dfd91a5568ccf6bd06", size = 54481, upload-time = "2023-05-01T04:11:28.427Z" }, +] + +[[package]] +name = "saxonche" +version = "12.7.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/05/8c5961b4133a59aec24ce28f1127818ca99411ae2d8e46ddda32d54cb79a/saxonche-12.7.0-cp311-cp311-macosx_10_11_x86_64.whl", hash = "sha256:373c86dea41fed4455158e40ced27e0988caf5cf5c0529fc341dee446f580c37", size = 36157196, upload-time = "2025-05-16T08:55:25.782Z" }, + { url = "https://files.pythonhosted.org/packages/ec/aa/ee1385f8816925dd0c180f442b3edacfb5c3e5ed05ace06af34575cbb83d/saxonche-12.7.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:62d1315a2bf71d187da54d45046dbf37c6b84940aae63a700dc877287cfd9771", size = 34813749, upload-time = "2025-05-16T08:55:34.801Z" }, + { url = "https://files.pythonhosted.org/packages/e1/a4/834d3b07addc8c4a9619c0f0e7e56070c555f6bb302c4ef3f95589693ff5/saxonche-12.7.0-cp311-cp311-manylinux_2_24_aarch64.whl", hash = "sha256:45abcb429647fd32758a3f572a0dbbcc6ec5d38d36868796b1f83a6e168630d6", size = 37764250, upload-time = "2025-05-16T08:55:43.42Z" }, + { url = "https://files.pythonhosted.org/packages/21/61/2beb2e9dd3d0aa63ee222f20ed415942c31a9dd162a0b7a973b1ad9ddb8f/saxonche-12.7.0-cp311-cp311-manylinux_2_24_x86_64.whl", hash = "sha256:262c284f772cad3a18dc06b0d55d4032849083c00049fe24c1e7e67a0c625bff", size = 38768813, upload-time = "2025-05-16T08:55:52.363Z" }, + { url = "https://files.pythonhosted.org/packages/cc/7f/9fd91e97b95f8dd0b0b2cd97cacd512dd359323d65319660833585f6b9c8/saxonche-12.7.0-cp311-cp311-win_amd64.whl", hash = "sha256:36444ebf37f825f92c7844d0adb7c98c950b60072dbd305771c0df473c469003", size = 35594208, upload-time = "2025-05-16T08:56:01.121Z" }, + { url = "https://files.pythonhosted.org/packages/df/19/4151e2c88f55c094f69144acba3608dc1e188897fd4e679400844933a835/saxonche-12.7.0-cp312-cp312-macosx_10_11_x86_64.whl", hash = "sha256:e8d255498a49055a950ee999a84dee003e74dc4338f25806b1e274fb63f2a4ba", size = 36162700, upload-time = "2025-05-16T08:56:14.914Z" }, + { url = "https://files.pythonhosted.org/packages/55/26/98d18ea61e168630272a4fd3f9d350684bcfab8b2f969a1001ad3b7a6e10/saxonche-12.7.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:285a09e39f0bae143b304da55254d18cc5206fd8ef938822cec5947be299adb5", size = 34821331, upload-time = "2025-05-16T08:56:23.37Z" }, + { url = "https://files.pythonhosted.org/packages/3b/96/78434940683280d0882b53ee7fe9a197f327b05432519d4f25b939c1989a/saxonche-12.7.0-cp312-cp312-manylinux_2_24_aarch64.whl", hash = "sha256:e861dc5eee567a28aafe0a1054585f3e8e9d1bdc9fe72dbddd756e34462c4251", size = 37691891, upload-time = "2025-05-16T08:56:32.203Z" }, + { url = "https://files.pythonhosted.org/packages/35/fe/688e288befa3e106aae30d2f1d6fc3080b492b9cb22f461a63cf6e05c573/saxonche-12.7.0-cp312-cp312-manylinux_2_24_x86_64.whl", hash = "sha256:fa55ba65344534652498ee6a1de86f32c2a40d273ec48eb996e62be45b9594c2", size = 38728204, upload-time = "2025-05-16T08:56:41.159Z" }, + { url = "https://files.pythonhosted.org/packages/80/a6/ca4177e7901c25572c6a32c1a5dbf842537745278ba765fb3f43943acbef/saxonche-12.7.0-cp312-cp312-win_amd64.whl", hash = "sha256:b2729d254cf023b08aea5bd693c052b2e67dc8112ec009429ccae6eec0114002", size = 35595343, upload-time = "2025-05-16T08:56:49.556Z" }, + { url = "https://files.pythonhosted.org/packages/25/d4/27adf067fe1f171d4f84a88b31a217d9d721736d28564f8b252f4a5bfcdd/saxonche-12.7.0-cp313-cp313-macosx_10_11_x86_64.whl", hash = "sha256:5244ed00c59528539a7635ae7321f3b5b1b9866660621e7a0bc7036c31a79e4e", size = 36162668, upload-time = "2025-05-16T08:56:57.202Z" }, + { url = "https://files.pythonhosted.org/packages/d8/1b/346d78712dd6ba24e0906d9a2feb705d97111763a273528679ea79762ac8/saxonche-12.7.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d6cf1c7d2ead0241e6305b8a2c63098bd9e1060960b9ba687bd9daf12ebc1e9d", size = 34819960, upload-time = "2025-05-16T08:57:04.129Z" }, + { url = "https://files.pythonhosted.org/packages/a3/89/5bca5a5a5e9f9df769787bbaa22f9ba9a7eedd52748a2c27099dd500f421/saxonche-12.7.0-cp313-cp313-manylinux_2_24_aarch64.whl", hash = "sha256:12dc57c1ea8df7565d0e304f59edb7ea30cec808305259e7e866c633ef151400", size = 37686292, upload-time = "2025-05-16T08:57:11.771Z" }, + { url = "https://files.pythonhosted.org/packages/11/9f/e40415cea669d397a2a00f6bb1734b7030c39b2609fb3bd4c43d61bf3994/saxonche-12.7.0-cp313-cp313-manylinux_2_24_x86_64.whl", hash = "sha256:b0493b4360ce2d0a2b0d4a493f0027650dd183bd504e837d3848c084a30e59be", size = 38724250, upload-time = "2025-05-16T08:57:19.234Z" }, + { url = "https://files.pythonhosted.org/packages/b0/54/fffafa4c7a23908f698f65b694817c03139c1337faf213cba6206c79400b/saxonche-12.7.0-cp313-cp313-win_amd64.whl", hash = "sha256:679a21212cc7a9027783121253c7039ae335c21cbc851dddff9f57778abbfc4d", size = 35594883, upload-time = "2025-05-16T08:57:26.203Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "triplets" +version = "0.0.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aniso8601" }, + { name = "lxml" }, + { name = "pandas" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/92/34/518c5c305bbd7bda58be7c9292c8ff9ff378894f15fb1666652f294ffe31/triplets-0.0.10.tar.gz", hash = "sha256:edf08b458a7afc2001e0db71e0efa6198363bdeba6f4b07ef46fe99a292687f7", size = 36572, upload-time = "2025-05-19T08:46:31.291Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5f/f2/328b54f0a2aec2311fa0d955a5b17e2acc2177a0e085b052b2098dde908e/triplets-0.0.10-py3-none-any.whl", hash = "sha256:2ab66c9036a20a741cccbf1170fa446947ded88665736cab8f7f1c2a8d0f6235", size = 21441, upload-time = "2025-05-19T08:46:28.127Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.14.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d1/bc/51647cd02527e87d05cb083ccc402f93e441606ff1f01739a62c8ad09ba5/typing_extensions-4.14.0.tar.gz", hash = "sha256:8676b788e32f02ab42d9e7c61324048ae4c6d844a399eebace3d4979d75ceef4", size = 107423, upload-time = "2025-06-02T14:52:11.399Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/e0/552843e0d356fbb5256d21449fa957fa4eff3bbc135a74a691ee70c7c5da/typing_extensions-4.14.0-py3-none-any.whl", hash = "sha256:a1514509136dd0b477638fc68d6a91497af5076466ad0fa6c338e44e359944af", size = 43839, upload-time = "2025-06-02T14:52:10.026Z" }, +] + +[[package]] +name = "tzdata" +version = "2025.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380, upload-time = "2025-03-23T13:54:43.652Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839, upload-time = "2025-03-23T13:54:41.845Z" }, +] + +[[package]] +name = "urllib3" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/8a/78/16493d9c386d8e60e442a35feac5e00f0913c0f4b7c217c11e8ec2ff53e0/urllib3-2.4.0.tar.gz", hash = "sha256:414bc6535b787febd7567804cc015fee39daab8ad86268f1310a9250697de466", size = 390672, upload-time = "2025-04-10T15:23:39.232Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6b/11/cc635220681e93a0183390e26485430ca2c7b5f9d33b15c74c2861cb8091/urllib3-2.4.0-py3-none-any.whl", hash = "sha256:4e16665048960a0900c702d4a66415956a584919c03361cac9f1df5c5dd7e813", size = 128680, upload-time = "2025-04-10T15:23:37.377Z" }, +] + +[[package]] +name = "wcwidth" +version = "0.2.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/63/53559446a878410fc5a5974feb13d31d78d752eb18aeba59c7fef1af7598/wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5", size = 101301, upload-time = "2024-01-06T02:10:57.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/84/fd2ba7aafacbad3c4201d395674fc6348826569da3c0937e75505ead3528/wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859", size = 34166, upload-time = "2024-01-06T02:10:55.763Z" }, +] + +[[package]] +name = "xmltodict" +version = "0.14.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/50/05/51dcca9a9bf5e1bce52582683ce50980bcadbc4fa5143b9f2b19ab99958f/xmltodict-0.14.2.tar.gz", hash = "sha256:201e7c28bb210e374999d1dde6382923ab0ed1a8a5faeece48ab525b7810a553", size = 51942, upload-time = "2024-10-16T06:10:29.683Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d6/45/fc303eb433e8a2a271739c98e953728422fa61a3c1f36077a49e395c972e/xmltodict-0.14.2-py2.py3-none-any.whl", hash = "sha256:20cc7d723ed729276e808f26fb6b3599f786cbc37e06c65e192ba77c40f20aac", size = 9981, upload-time = "2024-10-16T06:10:27.649Z" }, +] + +[[package]] +name = "zeep" +version = "4.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "isodate" }, + { name = "lxml" }, + { name = "platformdirs" }, + { name = "pytz" }, + { name = "requests" }, + { name = "requests-file" }, + { name = "requests-toolbelt" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/09/35/4a91181499a1e098cb01e04a26a053714adf2fb1c25b40fdc5f46cfe2e4f/zeep-4.3.1.tar.gz", hash = "sha256:f45385e9e1b09d5550e0f51ab9fa7c6842713cab7194139372fd82a99c56a06e", size = 166468, upload-time = "2024-10-16T05:31:30.775Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/92/8a76eeccb5c176ea3de3965598b6fcd10dc9d72a560e59919b52845327ed/zeep-4.3.1-py3-none-any.whl", hash = "sha256:a637aa7eedb6330bb27e8c94c5233ddf23553904323adf9398f8cf5025acb216", size = 101690, upload-time = "2024-10-16T05:31:28.702Z" }, +]