From 2e2db2499cdbfa63efc89afba90725c45ebb991a Mon Sep 17 00:00:00 2001 From: Devedse Date: Mon, 5 May 2025 16:31:00 +0200 Subject: [PATCH 01/15] First WIP --- ...UI_Backend.EpicManifestParser.Tests.csproj | 39 ++ .../EpicManifestParserTests.cs | 31 + .../MSTestSettings.cs | 1 + .../1sE9O19OT_X-rOWTFEiMUGNBYu8I1A.manifest | Bin 0 -> 53177 bytes .../Api/ManifestInfo.cs | 305 ++++++++ .../ManifestZlibDotNetDecompressor.cs | 73 ++ ...nCacheUI_Backend.EpicManifestParser.csproj | 16 + .../EpicManifestParser.cs | 7 + .../GlobalUsings.cs | 11 + .../Json/BlobString.cs | 59 ++ .../Json/JsonNodeExtensions.cs | 54 ++ .../ManifestParseOptions.cs | 87 +++ .../ManifestZlibStreamDecompressor.cs | 22 + .../UE/EChunkDataListVersion.cs | 10 + .../UE/EChunkHashFlags.cs | 13 + .../UE/EChunkStorageFlags.cs | 13 + .../UE/EChunkVersion.cs | 13 + .../UE/EFeatureLevel.cs | 133 ++++ .../UE/EFileManifestListVersion.cs | 10 + .../UE/EFileMetaFlags.cs | 25 + .../UE/EManifestMetaVersion.cs | 11 + .../UE/EManifestStorageFlags.cs | 12 + .../UE/FBuildPatchAppManifest.cs | 508 ++++++++++++++ .../UE/FChunkHeader.cs | 113 +++ .../UE/FChunkInfo.cs | 331 +++++++++ .../UE/FChunkPart.cs | 49 ++ .../UE/FCustomField.cs | 49 ++ .../UE/FFileManifest.cs | 164 +++++ .../UE/FFileManifestStream.cs | 662 ++++++++++++++++++ .../UE/FGuid.cs | 196 ++++++ .../UE/FManifestHeader.cs | 64 ++ .../UE/FManifestMeta.cs | 107 +++ .../UE/FSHAHash.cs | 163 +++++ .../UE/TypeAliases.cs | 5 + DeveLanCacheUI_Backend.sln | 12 + 35 files changed, 3368 insertions(+) create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser.Tests/DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser.Tests/MSTestSettings.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser.Tests/TestFiles/1sE9O19OT_X-rOWTFEiMUGNBYu8I1A.manifest create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/Api/ManifestInfo.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/Decompressor/ManifestZlibDotNetDecompressor.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/DeveLanCacheUI_Backend.EpicManifestParser.csproj create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/GlobalUsings.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/Json/BlobString.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/Json/JsonNodeExtensions.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/ManifestParseOptions.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/ManifestZlibStreamDecompressor.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkDataListVersion.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkHashFlags.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkStorageFlags.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkVersion.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/EFeatureLevel.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileManifestListVersion.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileMetaFlags.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestMetaVersion.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestStorageFlags.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/FBuildPatchAppManifest.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkHeader.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkInfo.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkPart.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/FCustomField.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifest.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifestStream.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/FGuid.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestHeader.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestMeta.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/FSHAHash.cs create mode 100644 DeveLanCacheUI_Backend.EpicManifestParser/UE/TypeAliases.cs diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj new file mode 100644 index 0000000..2d8847b --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj @@ -0,0 +1,39 @@ + + + + net9.0 + latest + enable + enable + true + Exe + true + + true + + + + + + + + + + + + + + + + + + + + PreserveNewest + + + + diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs new file mode 100644 index 0000000..a889cf8 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs @@ -0,0 +1,31 @@ +using DeveLanCacheUI_Backend.EpicManifestParser.Decompressor; +using DeveLanCacheUI_Backend.EpicManifestParser.UE; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.Tests +{ + [TestClass] + public sealed class EpicManifestParser + { + [TestMethod] + public async Task DoesItWork() + { + // Arrange + var options = new ManifestParseOptions + { + //ChunkBaseUrl = "http://download.epicgames.com/Builds/UnrealEngineLauncher/CloudDir/", + //ChunkCacheDirectory = Directory.CreateDirectory(Path.Combine(Benchmarks.DownloadsDir, "chunks_v2")).FullName, + //ManifestCacheDirectory = Directory.CreateDirectory(Path.Combine(Benchmarks.DownloadsDir, "manifests_v2")).FullName, + }; + + options.Decompressor = ManifestZlibDotNetDecompressor.Decompress; + + + // Act + var manifestBuffer = await File.ReadAllBytesAsync(Path.Combine("TestFiles", "1sE9O19OT_X-rOWTFEiMUGNBYu8I1A.manifest")); + var manifest = FBuildPatchAppManifest.Deserialize(manifestBuffer, options); + + // Assert + Assert.AreEqual("Super Space Club.exe", manifest.Meta.LaunchExe); + } + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/MSTestSettings.cs b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/MSTestSettings.cs new file mode 100644 index 0000000..aaf278c --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/MSTestSettings.cs @@ -0,0 +1 @@ +[assembly: Parallelize(Scope = ExecutionScope.MethodLevel)] diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/TestFiles/1sE9O19OT_X-rOWTFEiMUGNBYu8I1A.manifest b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/TestFiles/1sE9O19OT_X-rOWTFEiMUGNBYu8I1A.manifest new file mode 100644 index 0000000000000000000000000000000000000000..efd0bbba2f93ee4517d45b7080e71d062a92c4a5 GIT binary patch literal 53177 zcmV(iK=;24z`jH&0001&VF3V;&j0}Xe^b~$JrWTsyKN`pOeTltslOQk6#xJLc$}=d zQ*b2=&@CEcVq=m_%!zGd?;YFL#I|kQwr$(ColK00lk=Ubd+*c#a8K1)Rja$IYxP4v zG#)wz3=9kc3+(>^$$vC4GBRamGhtyiHsxaDWM(m7XESCoOMXj6-8BE+v!2T!T{6}spaTQf&5mqM^dMg($Clg6& zLwf^-|4%Y)`JXug2Ln5{a6oQH50oo_MM4-tx1WW=3Bk@$&=-!Z>)VFZh}KG%Vmqh& zCKSn4q=XUp4BqeaIl}IQIS}a`Klb+hPq{+5<$y-3Bwa%>6Lv>-suN4s@TEHJF5OHB zcloElyZg{nui_N=39OeLYlF+J^j3OT$!}Y3T9OWY-b|1m6K(d*D^=>1 zmo^eBgbKNQmoAl&$NYt9=DaSbVKjNhA|81UP3i>n|CqyZu+4d7mZ&R4ui~PZ++oP0 zk*l}B5v_LJM-Ob7L8-4goMrM=6Axgyrpzacsv&4pgc!}`0M?>z5*{56w_0iJq1%u( zh_r2zk|qQaj;i?WyE}1Ji`NJX%7(KnK`Ae=3iib%izYzR@UHy!PrK!infDNNl_^kW z5Ya=x2k0=u6L<(WiD&X6ME+Z+?_{U2-P(<$%`S5Ii7D7c7`$gh!!k?OU?DIk!biA1q4Hki_~H*RTTEz4chsa z6KNXo0-FaJYhGr;p!=qTs#oU6`vg1y+;f=L(+iOP9^uE~w-uk3z;anKbQM$mW|aEk2TW)Z#OGSvg=Q301MR?Ajy>piLWI1cg~D}$z4*>lu(ixPTV01S#3PPaG85wO%H~4udMpmQ zq(jz=3!zl=#kcOjuV;6Ar0RYx2~T%@oK`X&X%76Jlx@{=o~NPUSs8`096lku6;C|- zN+lb3k16>*j9HBG4XWg?`5ggy^6C97aos*wEZQ!bl1niy74t_v5`oIvQZhYG>KNFaC7TO>h7Iw1hBv5H%0zvH3diMX@OgYA^8iXGvn)L3MU!wt$B6l2 zYSo`C{2P%pg$r|~u%~aHrM%hLF!3v^q+1v@rB$N;>x&PSrx57{$$}$C$B{1yfad69 z{uW%jXcsb%z0RWHi!+}XsJc2;vcXv+Cq=pLH1ehw##*94C?I2rjDuEQDgl;UORkek zq5v*^g1bq~A0dl$@zCc;uIQb%HJcztn~lm#7+^9j5r>u3!lH!J68M26&WD>J+%BNs zQC_~q)+NA((3`3VsvZM96xXDQhSw+MObt{I62JYt~|REJ&_WqWDPJW)_mY zY|_3I;5bPkl%h2CQu&+*Z+m$PG5d!^I3EIG?$=teHmraaEbSS+oF6FbO%?kK0&-kG zxgV%6(%jAZ+ce^_mam=Jby*)tCf1uK7jCTi0v#C*#ela0ls`lPxJYU}Yi@|_UMMJ& zsxtUAUKwUX4hxFHlL8t7Iu-eCM60wdX)FGCuOXe?B#z?@270UJI9X30e% zh$}OmVsw;+v|`Q6l~``5IxS7P^Z`jzf4otPC0+9O2ck+GHz0$DW3$l+)%q$6f$tN^ zdy8p0ff|dXLTSNRNT3N7O@CEYQ6yo?v|1N*yA9OqS>>1^#;D%PbaqxXxEb&+Q|buQ zJWY>+@`kzBKA;F!*NgTs17)m)GG3ixl~KNZf{l?{G@sWc>mBU2;R3A@hzMhx_~){P``1+6g^)uiW4DSOF7c*ve&IRU+zp}lO9 zgB+DT*F{32b^9;llPl|bmos1uI7qQVCE77sce^G|ujfp>&o^)qx%Ezz0?q)N{wPlW zup3sn1md5+$?WlG)$MNe;_MUs`NsTe{F0v%#+o|4nX4p zljf5Svx-TpEVIJ@T)V!D*Ei|-!Lav`rv9N``T=?rZfb@{Luf82_Vpcm;fq}YW*)s_ zBoL9|E@3I8C^aJqmsK-j&?(<6=;dnW1XJPz;$AYzbD`tCB~nJv%6OT@!t_Yp7e5p} zvTNvHiL5w8>ul9Xp?Wn~!~_;|HZ9D>>RGWp5lTp9^V2UY1st{|>-6H-?mZ4+y@6vq zTnsjnH7Iyv1hG>`yN*{}oVQ4z4rT0;&xe%{IIL@*G71&3GML7+wfnHuNl(M&i=I{S zV8@M3S2=!?MRshvO679tZkk^ZbKGC+O%Q51Dm9fd5kutclkIk~C%q}O1Y!U{ZX=^s z4wJnt|8f$JgvA{wEY}gmJ0A<6D>i9{GiwIQG7XH2qcAp1Wc3dPN8h{JT_s8PFOjmu|1j5#gf%PG%UX!XXoO?Bc+vQYta&F3AC-%T5{$xy0DmMOJ35!sPNad_ zofKk|j|!Np(NJr13RWc|tgjYL$wPoayX#WuVq#*MRzs^8n>ytt4cKXWueMTYC6&>> zoBIgsQ*ij;`M46r0*&K478au12p1ALL7yC304rm*VP>hff;3GY%h;dXrJo|D{Lp~j5u_sw!G^?XS|-A6a(B;nIm zdDEa!NgnfG#a>jh6GcZX=;AUF{gp|>mkp&GC6bX0jwKRt6C7TP;J)Jd$PWYLL%0%w zgzUc^^n&L_8~?m++0x_SuILeqia@J_IF;ZkI{KL+wk@AyQDBzVud zmS?um(S8oehENG*=e7{pt6woHLQzDp?ydnETBP*2WUmg`FmC=euO;b zVb(zZU0DG1iUqJ175wcSaN(rgo3W5fFhf3Wlf(k{Qn=-z1q2D#hsN!56b0xHPQ?Wf z(a_6P{)pj_yzixzu*NAYRqGatQT_=|Myb#vMN7M0x-=>q14q9`y=q()JaF%5Az~@j zQBiVXJu!@?52LI=I$I6M>N5D~f9pLf0*ZlLETRC(s>J<9{ndFHQ8kU^WF>%rVrYgT zBx=~92!Y4Lz#LGz#$q0iHNIvsADJW9#fEVA6`^4u!%lDTQ4=Op@0U!JjIK$%9d;oG z^XI~6&!Sj-(o$v`)pIwK`zq&;n_5t;OFN=MqDBcioir?%1&3@oQf4QCe{Q7yvk<6Z z-WTBKnlYXiK9z1FGiV8W?IoVZlpkGKMYO-7F)&HGAt2ck3MPY(7z#-%1FLY(ODLX) z2I_#LHU5pz@7cerWhh)1=t=Su?)N7dzKY6+gwqrXhbuRo({@;Gx4F@Lq&K^$UTxae+EuMt1qpqU+rbf z9Wkk6{IdA~kcvHY!4u2_9< zR|X>Y_#hI1)Ho%glLh2$KVh9h-)HIjraa4w-l@I>GC_RYO(akTC zw1MEbh!l! z*pAY3uFZ({s&ud;NTFl>5_oeG-+GeCEzUYdR|YHmdWr>CoKFOX({;5i;tvd9X>#*y zcYBF+nD>)^OZ?HkXfZsOqX(0WNHu+WoCzvP#t0dY9wCv4t7dG3i1a&@{DZw$L#z&{ zRVOLDGOH>3fTDr5k%pA%7RucYpuL&oH9T;qrK*rh&Uv68tQF8LvSl2-HJKKTtFQVV zOsoW>`=z7bHQg^=ku^loYwD=-qO>?2YBVP0F9NElJ|c6RyHQ4jQ&$mfXC}KLoBh-! zk7udh$r+N0Szc$@$!D#NU;!5fm&p-JpfE{L1ZR_jQBgSPxpWoEM4{w28H?H}st3EK|_PDcS7 zAi1jW4RB|y^rg?xqojl?7s+<~h5qC|Ebjyt45tB;LZ#9K+}xzv_b3JM3|ZyE)-I8$ zM|}&ueHjbuhcdBXf~u&$W?o@#te$79(IJwZSbr!q*C(t}resT~ zq$tHJ6`GOIEL644+0+v$>Un_`#Z}fyX*&Y~#h6X`i~LeuFf<);m!q#E_Zb|sTH>OC zO9~e6FJh3Ls&op9=h32poH~RvFz%sN=X%jx(@Wxwe>O99gvJQuoS`G4R_f&<&|?1% zvqX80i`}@kM@fhUE~4Cn3(I2j!v@S4@a)J|VS!P5|1EbV`@BT8CDD?~$)eviH22|9 zvZw!Qx>y8ME3-(4od7ANuA|o`L9OEcR^0~~t#z!j$;2(g(e9vTR{lluQXeRP>PIJ8 z&U4Wqu}*@T)nT-!KBYw{<$1ZJpiZJyE*6(i56(h~br_kV0Y8PMGy5uy9!6vdbZpUz zpAQQjvm4`5ER94Q2vE`~A36QZbE{R|r7=6od<2r!XWfN{&*E<2sv%i+mW&d`ULl&d z9rTButftm^h^n%MZ9lkM15>PL-*LrIqKKty1Jrr?v!caR+Teud z;I@Ne*lc2{4s(byO432{5%+M)(=USs*}4iLx&g9+)a~&QhOB=UnJBLW@#rDbQ#3o>gHEWNkjVp6 zaP~(yTOGg#wfXbb_4UWzvC*YmN5=3^hS?Yef%1=DF*JF}xUrZBavO4Em5IAYg-4K7 z1!+`a+EDX7%iY@-n3leHKy$a-S$9jb94S~|``V63gU544>Ibe#8X}4*u?vb+;UW`> z%E9D1CY_C3+N_skc#g;*gl6`(OB}d&p#i6ClbyUuAAju8K^(P)v@4fvYC%Eu0BcJS zK^16A@B3S<^nQ1P0UjOM#~KSsz!OI-70l&9uA#hsTx?_Z=2T0v7-Ngr%b4T0fes@9 z(~2l4{{9exRYRjc@Dt~f47FOQ(ZiU^{nOpnD)#&UN(Uem)vq|Neevom7doA6x2*`s z0h1!s*LD?f!C?&q8f9M=H_cpqV@ z)Y$7}Jg~ZEr62&B4q4t(D`y{xdOpH#lwCy|`3Jc?>A1j43*Gkrf|JTsHu9O0hW|sk zV!T)*pO*&Jqqb%v2lJ%rt88dksa6YzaUX{oJ?nGbmh^bi92iOEBm>f>&LE%u)W}|H zmr02Gw{6R@(SwuHMLs{7mf(uEOj;v?4*Ay6xExOPS=4~7FRmNJc5%_(S!ja*3R>Al z0q9JXIkU~ID(erVPJhXO^2GkH7mw1J4Z`(5A19z#WQC1J6gOsLqA^ER=X$XULjbd} zm$$(>HK!4_1O+g_Q0{xygayq2i?iIIomeC&7Cq(!fin6hFLm9_KpLnLKEuU$!BhP< z86#Sri&3K2#YwLKOzFO;Xn;NN&n}>ZJaK;y@mndT5oKjJ!x~sbDJ}eBTQvetPH9G4 zE-M@`=pO=+5dLo;z^mkg)hrTjsFWTa%J&!yo`5L00U_M+oJT zH-)N~0^nAT7SDrao1wNski<)g{JVPiIzp^@b<+LE_9ZI-Q05pvAKg?uk*Kd+u*9nf zZDX*4_YL#^_#SCBl}q_2ant$bi*_Vk7ct>sRZ4_3p6s^@w&Y>|ronaMBb1DlQTCzV;gSqqoP zWfAJ#@RQ|wn9e@Omcl_#I00nrmbC@xfyA*1?nlFT=Y*4SOYM{uGt*?YdNX9>3nlCL z%rw6e6aC#|Pni06nWcL6SR0TaCQ~0#s4+W(`V4z^5HD}6* zH8jV_G&@n!Zqc->9iT-Uk2iF_2bhEJv2gl8E8Y;?xmD9~L5CH_t4J+kJ7xjVd0*OX z(896MDmKW4w?bX)2~kJ9P2oI^=M2 zDr$ooBtjZ^0cBb1G>9!_n_imh%8d2Azb<~Bn$jL*QGyDr4P@QNO30Mg{n#YA|RU4 zu{JSDAbDcq#Tp*F0t5HYWxd@MXaXXlgeo+-61GAD>79ui=H2N%CBkYoQkWdv>`|rG zYdvT~0WE|uJH7ZJNV|` zw&+Vf6odn!bS4joLuM($e=J$tl~foE*aG8EU)_JJdz`mE2p%#emK{7W<}Fl4HE^Fi zM7-!L*RR z!R}?}GJ8e(to0Q-@oFpGHCMod}{z<*(9^fsxu&L5fQt=T|^9;aZRjC1=@xP)Sf&9SR^d4He!=)q2kJ z++8bTe4z|0kmZ4ZWs|WZvwobXl2IM7^sH4fB?a;}qKCBstKtwYx|OD)nWA$A8m+x4 zbA+>XDTw7l)hI6Q7xqV5o>PuQ%-ro#p;!~+COIiQ*K`qn@I*DzRcg&vhrl`P(ciYd zO^Usu)$>uASmEJkBt3)`ZGF~am6Mr!1s%y@fv;E|_fes;t&wGl`XQgz>ZDVXD(yL9 z`5IU&-OR;ItT$M;ANZYO34MWF*9B-SmVZ?vfiSU>x%pLhVqgkwr^WB4d)`sP5jb7_ z^mUrri1%K0-+}tVO-?YM&z!E(TP7n~(Jd;{t&-TjinX5dXup4VStc|A+E^U00P&Vd zK#ycv<786tz+sSsk>tOVD!N%zW-MRnaQ|qvEBq8TyYKwm`6Ln zZ#O0!IhA+BE9H)&GIaK{4aZTkp^Uv_r)uI=^d~y>rVOuW990D$GUL0yI+U+-0G_9m ze;0W%-w`-g)s~(qK~IEG?GHZK>>CI>cGu%u32|GY%xq01(dD2s&R787B3 zaWSfrA$dXk1z*_x7?s}Z81VZJd>(-uy9yr9w=5&>Ju8nssK3Nf{%YAFmbjokxkA}lo_$Zp87lcM0m3PdR`CdSi5fhWW z=Ymyf#d$;QygCCqnw@FqwMy1A;37BgdJ=|Tcp*WieQsuO0)pgsz1aSR zFWSKnTs2Dn>@8pcH_R6M>KHt5i|DqzXL_)P`K11P`o@XrD!XRZ42W4BUI|p$^q)v! zBSZy>&fQIKEYQi#MOW1fWO}&F?q-m=W8*lZQ7yoITFPZi_7MS~NS4gV zAIVd_SkSG(RrPk@3SLGOhq8kCZioIYQ}Lpl2u-|s@(}L*A0G(*53m>YHsqAR@asbT zxJ+iBZ%V|}e&Wh{D{mxo<*zMovz0WBApt_E-M_>V7o#wPyF)6 z#WFX_+3+660!naA$T{vK7FJ5;jU*=S61O4hos{OE(9*uRa!j@Hr>!!+aTHLR#B%u)+I4I>{W-0JL(PSRGO4CC`Lp`=-e55-@-1{GyIRFi+L&=5-m zL_Bd`l$&qJ9tVo%hGfO-aG$su z>@}WAvR2$+A)te~=nP0j(@qAUZY>cAgzGc%fV;}40!}I!zK+qFX1#&t0l1dJk!u~r zq<*bN^!p`;asJ4e<%>CYuYpz()7yx4ARzNt^0Y!~mxN%nw>`{M1=YN*EN~>7QA*XF z1FQ}HZ{pe`(_rCQd{@Jn6Ewqf81)Yu4rpiMiFCO|O7LT}u&zIGBXW-F2eZ^khSyEQ z(TdkM?j!z6+|#PSja2GECThY_5lKC~ufBfNb+#e1Vj8|7E+3`29H*`)X8*NPKXLZy zS|H+sbBm6vP#>qIrE@{>tNceD9U<2Pe@yS%PS9v(c=5uyyU?TUHR43njin+Dn#MXT zLy|DHSqqz7B+5e;Ga)PHtiGx%EA?%|A`fQzhVfvT4g(bTk!xBG1D9}UhMqInv3voX zVhK0gJcY?o`LfGjfNidwXQS%tP`Fki(?<8_{IiF6#k`%2De}PjkV< zbS2xDTSrUbOSkptpk-zXw|T#MU#CR_GM`pZrkCmTCy|6)Lg3X$Jw>B%itXu~sWxR4 zQhwGNMudcV&Uk^a>}A@maAyu?9Ffkr9CsihOvE&={Q!(`B*SJBv@e4~(4bVnq5t5n z@@K(^quViW-;+2g+KM|gmULvPtX_|-Rlg%8id$z``N5z<&&b)2taHT%NAIPMPD_`(Es;t2^gAL`0`BRVsCOObndFMO1EN9<+= ze~zp!KA?eWO7=jK?wurDc$UK`g8LofH#Q5_-WVu^XLPJF+dBO;F;)ebtH*L2jgs)Z zya$NB3!C9bzGJS4!gRPJG5_T6^OJ&ta6d5j^A zy(Fr+iVEgi?{^Y1VV;FnAZ3)F&qwS=30u{k%wxu4_fv@{TRW9e`}<|8geeS5Skg6V zdU}R;7Y}yOX08m3dN8o6DLe;PTWlo1=1Hd3z^Ep>3-%YYDHfs3 zbJ$~)w0ulvb8KL>T-@(UK^DD4UT-TS*ZVWxz7kf3BtJ;BfS;XJi_)LYB@{9#POSke zY!Z;n`-*$iOKPMo?WUquAmooSIDrq<&-VAI!>i)aCqnB^o5C7>?}PbSn1QQ@IFpeA zR35kWdqgY&#^*y#$k9`lF3^-wUUG;gCfas6IASK5Ic!&2AK1Kc#BHBiwgj}{Xy zK?J{cP_tIcpn{pff=40c+UBSR7mUaYi77W`^Ojw@&uECZ+Zp2XTraF@l08@m;USlc za82;XpP!lu`Hm*YiWOL1L@Ee+QLT^C`VX;?1%O%Xt(4Iwa_;<154Djxdoq`I4U z@S2vTJN}!#t6|ZL>Abx8qXCH9A_%rQo;l;X-W;}mD_-E7-ldGCxfe&gUH`^N9ey=OS#ltI^0}Th56QPtaqf0JI zS(4b#e|g8$qF%>(RRLx0GiCsJRpN_cW*#$QR*vOI)bB^g1v&7H@t^{Y1Dn)|e{Dp8 zf3MC_O4~r!s z6%}xtTmQ1so$=Tl!=BXe`*-*l-66;*{@&O+>c8KGL)&==nt2#CO*)w_#53v={9!NL zs0uG(-@m<7Z5vSX1&N&bQ>PiQC8?EpbE7FVhqk*c2SZOdXPnv=8vYV(HL(4MBIa-n ziTLq-ntVs2!Ab1AdGyE~`TcnBQ-3NfId##`|S+U+0Xh2OSrG)OV ze~vZ~K{A%7G6ZI zPC`P>bCeA+`I2HPsroYr?nQzH6+TdzlWa9>M4^82Q~Jg3rxl@@J}`7AOk*R1k|=2$ z-cL{<>+j}Z1XqZtL3B0@Vt}P3+w}L;^W{$Sb~J~Ztbvmet{x04c1>gnWu}vZsI(Um zv@mT%?tB>gR=F%;ll4k8Jq8RvcHK!~D2!z^nMqg_5q3y_Cmr*oNy81Fc-Zh77~y;+ z@^K**71zqY1aR3~acUaS;!v<2nm87W`6+AAIc?6UKWEyvdqV)!yZ#WyLKXB(xJx9# zcSbQ{=u<1)U%y#+UDtw#s`h`O#aC`WF=N!EBUojBm;~onMCC^^R|D|$Y*?1I9hz|0 zHg!p82kY&1q?D4^d2`=8l4o}jZzzBDJdH~KHgOLM!c+N-KAXohgrE$T-XVNRiR1bt z)HeVJ>n5ZHZxZMKX|V6r=DJ_hc8(K=h}5Tz>m||cSU;o^gbYbBRdPFldM~>fjGU+o zf&FS8HykGKU(Z)fG=XtGoI=VpKO(W5Cy(HHHQS2K+@+|28qWrS?$gEhYZKw7?%2vf zP%9R_K^sD+&FebDLte;vV*o_hxMKvJK1Kc~uY60%;amB49aiyGuGWOJts{5ixYXV3 zysf~YpBT;lh5orpePK+_o#}pg;%gt6ACS))t9O%gH(L?!2FJ7Ed0vua4$UkF72khD zv{&OZ7-PAf6DL7*^t%?swS{>j2_>2R`g=%AYfI;+z@mR9+}XeyY8@?`S(usop>~%b z*Zz194mJ9e&2t;Jh_Lq;5Nuyy9qkpQ5)m`7hD3@1rk-uhOY9<|}zdlk*t(K-5own8eFb8Q|$c-)Z72cg^UnMm)X@g`F2 zGi{O9>gS=iV4$c8ZtTuF(5xTy+kZuTT!x9 z*2j$f^z%h-U+Cuz5Kxi#>EQ7-DR1SBp}2U#X5R(#oU2ANGUfS-kMVW$j*Zj>@~{ZN zR-a|Z)`8o5H#U5SIZKbw?CBXS^BI{-u*O%71n$X#=Sq5iWYGhG+Lwp!DcYGVNXaq` z&5W9OoH-dIy*<&<85P*-Hd{v;iYp;mHU}fI943Te3G?tPP;&)Zav6;j!XK?%s_(P3 zt%FLvpZ6@-dQFxr5HqOqSj<5J+hoI)%r)wnO3kc=40rB{h!Ced1zl9D58L82Gm%yz zI5>wv`Co_iyZ!jn^^Z1Q*6F81B=~D&XL#YUqAa0*W3W#hki(V|G)(^*ld0|86uOCeSBgLh3h@GW zBB`$qp=_FPi=E-WJ`I_{dGTXZ?MoaggT%_w?$5Rm(#RXHxST6n7!oALyXypv!pdFq zPvTW1Ud;Y5qbuhSpVC5xnRSoqFEcO&R(lA|5aGJJHQIK*k16k=kEETi85l-;ZU(Npm zGZ!MPNu)>)%eazrWXG`|&JXY8ImN5Qd#IJb+Ju_Rip0~X=>_ zG73zCCAe8$3LG6MlZAd)O@wf3GS>XasQKaG_gs*x`V)=YFh>|^l+>92O~-D?#^`(V za1Z-civnx*d}1W&x$l%AxIs`V0HvHg@noTLW3Lfn42>@B96c?mE}4Tl);8ZsIUrn> z#r#!3Bwd13R%H)G(ELR)^Z#~?*ZxuP0R@EBJRQ89ll*t|Ot!i3|8K6_zi$s+xYhz& zrch-=p=(7lF19icE095@W^I`ma=MhD@+~7uJ)+Mx<7Sf8>S7jD(N!$Pstfyw)C?GI zKwV^<31h;JIq{)(yXvlS!m*|Qnm!q|bIH+yPT|XQ=Fogt2b+wop(k_+*zfeQdDREO zY^cBIY{u2Jmxr>6<%HU*2GLg0NTI)V9gR_$LG~5QilER>ST<4FG|wST0lM>i7Bo6( z>ie}6X54YKqta5z)Sa5x_|+WeGjmcNud^~7E0F&1{>o^NMewVznvw0cI3{*8nUYkM{5!dHxu}?;1f3*Mh85wDXdI(xM@P#o`tA^;A zcoU1;%Y4d<^{$fKL#6PgFkZK@@R83vF5+@yAFTn8g%@G_bmB8QBWK~R5Yj?^c&n#N zZ2EgYYed{Oz=m{ClW#F5WB&#jF;O4if6Yi4T6#gOiGFK+95E}{If(W5>TtF*gJ~Ua z8EBXr4IfQ+Zv3;>%HoF|?(qFHnJ zYbNbXk3WFrzUhZfgNlY6zc)bEh8vth2^#E$we)Z_t{fm z+-$D3ql^9KT&>4(W8>&=c)O}$Ru!&%EdQY$@{IS4%K7%!nKF&WaLixnLik`7fF!zg z1BcW%4Ufx_fG`{f?Y(Imqv_|3<@h!572GuU*hd5Pa8JNr9Cqzm%LD~)?D-Ez*TKwc zNuIl}T6VvQVa5e_mTD zYTt_tyvWt%Q9l|jgheh-8BE#@t;eP43_LN}c04I(L7}Cd9zj?=R-Q4f8-IYR!)^ms zc)rxTyw9Qj;4nXM&HKeND&v*+9v1YGFNK{B34|A|UlHx92o5+hX+bc!q|JX@u#SpN zUbdo_Yt<+M(aLRdj@bo7p0pGbtjwG`FSD`9`>F*N%VwM6M%Z@|PXvN5aE|zo$0d%> zJO<}C$OqH-Lx9rC#;JTm5D(2`Our;hEF`8wtNS^859C_%Vg5m`#$DGkPm$$Tl`Mxm zFg+&e&udF`TdNXvF9el7OGv0JqIa~k{T$fsV$(pD#M=hU8);VPy|a9=u3 zpl?K5W%IG(w!yeUMoPDll1tuhX>(cK8+GX3W>CBa;T(i!{caoDd1d^buIMT8`Wb)d zF&|0sM!grNlg>7OjTP+*zPA0&lNfl$$Z-2xYVZokWk<1kC2UJexLMI0*XPcoxbcT! zO#stwbc`fyhOc<08$(%938E{%b2U0|9agw*hWO`R!#sQX)>iE8w-i5!6&ud$r?%uu zO@S^rMilJ!ulRxq(PF(eJJW^hv8i_2n&!Y!DM

=MW?W%5IQGcZZwK>z$ex6Lm0BD(6%X?Gh@DkpKq<&Y z#9-|vAvuWm^v6(nrt0s9Cx&d=cKyxkW99}9OJ|;C`+a`Y($R-n`T+79sOh2_B8tQa zX^*s>t2!K~P1qy=PBduHK0)>lM(fjW7}Ayg z-JYj(WHnH4A;%{s*-V)>w3Lf1+^`POetsh^7T?6qSG0gy719(TrmH(%5U!-TepHat z8XjnLLVeWvqqZT{BD`2PQsdh-&UkWy$&W>5qAg`IpcmD+wTcuH#Uxs)hc)W-?Z$b> zhTT&_0=738PkkrkBG@YlFipIE;Q+guz4M!s#+;6aSeBqM(I#U^0QmTKaG z^01-*L}KJXI*U$u>7rQ*I8jol8@?lAs%cXo)z@!uyU2=w(6hDYp1@RcbXyIBP>v&v z#d%+hZv7}WN_Rf?fZ|wk!?A#_LI-B|{s`5%-+uG#shg8DGO)xhxHPr zY#cT7o(FAnW*@!i{2)b<`OITLM8vl|?cA}D$rm0klVXvV1LpmB-_-V$^77zqIUaIz zkC**4MX~ctW3u^=?;hJ|SF-9;P>-q6R3mD5q}w?(e%C6d8v|O1cfQ&HO2T_&amlMF ziC`Gf2tnJ$ALIpqud6f<0k+(#+JLD?-ZA*YWg%I)0a{ zWRG}IJQp}s=)!gX9(HC;MNZuwt^}Y{8(Upj#tmeKghYIhyq_Y^l*iY~Yn$J+L<+MU zW1bYP{QdHL%$AJkUf-Z5;n*+mm9$|O{$`)|C2SpbF^=IUR%_X->tkP?*L|uaqDxlM zO!N4Z@g9RaQYz%`;8oxDadY5ka-&a^{IH_77A5b@BV7JDLwPAm zDJGnC-TapzUW3XLrKkV6du+DBB6z?ZwM7G!*BcH>Wo=Iy`4#7KCooqP9bJ1FYInJG zt}c4@xT~b7*5$RXokHV zytpBC>o9>4HoBV^VM&~VR1{rv4Zpl++9CB$fg~u;vuW8d!Twzg3TB6^7g4t1zu_Yy)~hMyMoCl4eGZ$XWy33Ea7nd0T|UbTmR z{~bsBFDk#1=7>0ge1e|23IQ|TIrVJ)GN2KBGyoRNhj0}-H)FPemWtMv5l(%gjQJL9~atKMss`;U+De4HE2oMFK${YU|NY&Ms+`=AnQ)QI9}MF^e7QDw*}P zOj?4}{jLL(_QxssJEZMvAND+mq;55nK{;w2j4FL25$inF?_L2d%v+SeI>4XcG}n4+ z{$U^r%D+To!uupv4ZJg2kv^HD;$cI>LB*--qE<5F2eC`y1Cs7Xw189GJ*_)a9L@!$dK@GDG zP~Vkl)v~?k69287H}qUJ6jTuc!~fzou-qsAWeNMxJ_Qus`T!Vkh_1a89k$=M1N;rw z;aE#CV`kbpt@e65AH24{RygWi9Ip0q6RC(E_^sd($eJK#+Q{}!Hze6#4OMJnZ|j;W zNSS|XIflxNxcR=%suV+KUfWZbVC7|wvI5q(quhc+wd^-1N+kmm4(`o0pHkwnj8mJt z?XanouesG?e1m!{Iyrc|u_X$$nz-|Q5A?jgT|#9?oNG9;%CtRDI%@D$k-tw%j*}U*tKSiui&Mc<~89n ziW$OET$l-5`+CbpGW-y%0rm;OG>fEE>EXx15DllKkDbqi(Pht$Gp%}~)DUjJu_8Yd z@tC*z&PYE_<|)m(PV{PB+aKlZ>m&6d_Uh!06qvj(i z|H_+DV`XV-fL)lVWG4V(`;9eu7`>U<1Xw=Kx<;#uPOvJ^nyIs2TLbY1LuF zkC;mOX-i_xQrNP0Oq~`ZAC!zG$+l#(p(F5FBvbcRo68BTn5do^P`WzUV$Jv*MlrbwG;0tg?GnV(J-*$hx`Cvw_xZYjuC=4rb(c9%|lh z?05YdD(rVal{5`2gfnz(T~1$B(d|fDpZOH89)ToL-W1bdNe@v+BcxIT7mATV%F|r5 zH`5PZdzzKJQxPx;zVOmgB99SzX&^E574XE}Ku}7&TRfhK%j2o(f3f!_;81>l{P)Wxw&Wb(aH5NkKNE_c3I#IjzyX>P(LaofEUqTk9J746VJ*QSpoVSo_o7FAgug8CGRS%<(K8J zTvZBG`_daNDP~X?YWLQ>sbv0s?$Qg?4eR-bT1AY~8EV9LqC<`;RTm5@r>D!mz9UHf zpeayAWf#$fZG3(<4aTWk(3%#a ztYiaq@28B>@*py$%^#H$m2 z{O&!oa&9I&%rcJFH7&kB@M7Z)+T(B7Xpg-}3x{WA?%CyGp&)mT47M@f*0O(P`aBhJ zW7^BRls$6YGovR~h?6ZXAuYXWT_Rq2aRLS+1*YO2p#i0^6i3FS_}o`fn$lHSou!{h zg|{}gmQGlvKTO*Ftk=VB&{CP$J?LE-(@>+jd+pNjLw=gA!`CBiT^HW5@kl%v>Xa&S zB=J0A@^&OIpD6Izw#w1l4z2YBT zRS&^*WF3q*We|6hnCBN-zYE(^nt489eC3twLQ0G0qNdRKwA`a{FPH_@C={p{m&eZP zTdS|+4~8~9eiC}{M%86?Xm~(aqT%6{TdCI#rp;V8M^ha;`8kg0>~Yc-PjS+#D7PfP zyfG`b_S23>WmcZjw^i+aqhI}CkLb&xCjtsXEtFJj?l>#ZqGh~;1@iyIZV#A>$CKg}H_ujky*_#H|#~RiaV>d{CjIWk# za>J9sXjr4Lk9;y$oBM06=@$mdde8TjZ@e7k5G^W`*!khrTcw%3pC+1VR4@9(4x3H6 zHd3j!?M<`z+R{q#)NTK=^I_X8TE_WTZH;`?ipGbyyCyOXoGCk%q$eZZ#wRQpYlui) zxauWX`ZcVuGS-n?@hjC=FXHaE9Au{?kG(jb^72I*QqAF96w&ZfLS||D*VdaGgd2CA zU5T_Hp_6jBX<$g3+{Pn%w@sdvxt2yti0i|v6?J*TT~)kXr@Av~BmNLZJbja?j71)AD;YPP`ZYXuvsqY-Kjou1ZZh>NrJl=N3;9 z?h|!WN$!f%Djdq~^^fXXb>Yxshh-KkcSan_HE9f*GV1$j=o6hgFPl9mPsDB;vp>6yag1wm_(5Bo$c;t$u$Y|WETI>S+m0NCku;o= z8c}@xAvS8kPJrz4nYz&F*GoPhHh&VAtvMxr)pef3Mnbq(if1>HE6y(A+Nv$$hIUbn+^Zo`gg-ty&M%j9c~YPV|R{M*~b=S4QL-kW>2 zAeD3Yb=pM2Y`I;!mgk-Me8I04pO*_~2UU2E7bx3I#P4&K4aVZ`(vRwvD&9vUA?ik)kGl-C#65uvc?2x2L-Do0ZCvv#E! z8Fe0X52vp8`4lQg9du@n=1j{IS3%V8e`~7e=tQh3dn(d~r+4^NYH#qEK&mj5-11>* z_9f%9rlakzXky=OUU9oXO)~uDuyabY+O3Md9ZJWq*SKvR*-X3pP;EW8>GT=#bGH0% zsrk|ZFSDNA<7qehcaM&m(=1d_@`fQ zCrg867*t)k=JwK9B1)v3(&bLQ&%6PROEK!4FV$kVE%|#?qH`;xY+!S!V>WG@%_pe` zNvDwvYHBx+(QT}a5~rO$_qE2Qgo6K5(#2z&#JG1vLDD6UD2xbjUNU*YGb1*3^qyJ4 ztdfZ>F}wESDktNkiH%R?Z(52P z9o(%|EK=r}oO)+SBmY@GY>Q^y&E4=R)tF3vRysj+&u7h>s|Khw&?o zPx+HRhOH|FIm_2=qz_be#T|a~YSX!E2X-4h<|Mo2dIMfm%GM`Y*DQckQXz^ZiDVBwrF)<4w?i3S9&dfJFad!d7f#{r9v=x0^%}NB z(J9a@TgY)`8|OqHpSI~`y79U7M!eA_G7F^!QLcy#x5Dw@*V)qahjt4oK`IttADSPg zSAN#wkIKH4lw(jIy(KsI-0arWZO)5x*^jmrLo&B;=WSQ67Mcr%o(z3Mfh^cPFjZZ0 z`9vM`j*v+rS+^0F%N%Qt>I+VI`2A1yANCvM552p1543?h2`NbBk7!)&z-l&Z*< z29@n+V-VS{1K`=G(8!Z$kPs-l;d@q42*XyMt=jDCPf$;@GJNy%TJT#M( zuO_Q^PqKT;e4zQ9^yU3T#g=r62WkCPqf}}u`-=GU*?maIu0>s6R(U9}f4sNB#_i70 zZT=6bGvn#6Mk0ydQr32t_t%UWFgj>NZzwr6Hl!<97o{Fn8L~P3Qz?^Yt~vW?XP!^$ zRgIh8;rt(Jk*<5PwGN7QksNFE5k69TR%Ku6jbt7~c~G68R7b@&K|fgBX6?XIDaFu$ z%PdJRy5qSOGZ*IlMx;27T;Y3i_t+Vm5YvMxR{JGVSM~>A&5LKA^^5ZxHyjxea@@hE zNoMl0wPxvqbVT^|yyl*cx!F&z@@w=(U|p&#Of45gsSlJp7Cb|lS#un!iRd%GUr9vM z)_cESoJz0#ZC+Mpv1}2q$JG6FmRW^qikNoADkZWoI6bQYI9m0^nYdcYAqNB8=Hx@rKS3up9&Ls22q1xBRGsmH{ zl*InL0*X!NTJc83rb6Z2w{-=s?BtcDA$?#c2zBL>qUzNTSxH#9aczRnIpNH!OIriK zwlwuUn@Nwz_fagY$S>@dsBb?#ZNz7P(rKZuwuwUP(8-lulDqcx?+6hfK892~e)wu+ zMNK%%SfN!=iLFOb4&{mW{$XB0J*RB%YJ?g0!N}kxGxlsl!OaD))6<`bjUO62ogY}; zGLl%RXlbGnGE6aGx+&5bMF-+ZvMRay6C*@)oO3t&pCl>bBs&|DO0n1G|BJiFV$(eW9}P z{fu=R_q>@X^(8tc>Rs+ABTYNrk6iMyb#OiK;HG3zSjk|Z@)sC|Q4eu77+Ly%`rs5ITi?3Lvn6wv(cG85NKx{D0>vk= zP|KRFL!_RUHd?XCoN$|_O^sk0nIk= zM+nj|H_>IT^8)6)w^(;%r-TRGXI329F_nYdOV|E9yRF@VFW`pg)X*E@(UUSMtWJ_B zyY9BV3TcMSB?ck%(onOA;LGx^tpkutnmNT3=6PNP-uVjBh$h#?+S;q01uI%=){Kw*ctX_xrW>2cTHjayiU0-|wzYfp+%iVpV`(V+ zLSMh4rLUK=AJO%?0@qyvTh6#H4enJ_h);cyqbqn!oVt@EaQQH^#j+ag750x*eGx?o@y}LL1gykj)xx8a!S5|NGxd-`&IK(=fCtyW8 zGMBctekfKb9H`$=7<8cgpv~8T-F7LrE-W4L^R_A_=xgReDy5sg2%mPJul9y^(VX4M3#HXOJ!ZoX6nu_ZHCPo9f2-wh@>Ne=U2Ko zOUWC%i#FDWm2J*OiXA1-3z4#cYa3A$J5?;#genBtd5_-L7Tng!vGjc3M%#{scS5?+w~mlbfP%yWEP@ zUN=ie$8%rF`gn6?As<)5IYzopiPd%0>@3&!Avnn!a)jn}Mrn~ct_9u5y zSxQ)4X|u!4S1Q^bROe0G8)1bN%U%)kv&TIT_vF3JPhcZ1R(_dsx4ce}VN|X3@#jdE z3vY!FH@;@dxx?1`J}`M+|MZ98k^Kw%x%=5utqt#4U$mTvnALfy#^KlHaAI@!8;hVn!D-dqMt|IK+O^25P(7ZPPVV!J}Cf)XE8 z%h^Z?Lte{0VOnr@hCaSJqv-gxo4jLhU-1boXuDGNPRUCwR+^XChF&IfeLd73o7#9i zI{yueQypB?^pI$&}oCPS}jTXRphJ&9|ANJHce@~2z+0ibf zh*AG^gco(?3q_yqs^qTE#}PJ1?kuJ@Zt)K>*OjL+bf(zO&Y{bpM9;;`>1Zh~&ApM# z!A6_R!eq0s)n*1efld6Dl8mIA%oMj;^D~>Os8Xs+Lan%E+2}P{WyG{8x2bNYW)?CQ zP*zn^+(f#8oy}EML~Vzii*P7^ja3Qi^~3Il3BNf#nH<*iICWE_gSILM_`WwZs1)Qu#0xyU%A#VyEX z6!^9q81EsG+~#V}YrjoWQ%r%I8zS$jv4g|F(1b14qz)_TeL56%6y|$Fq9xYoVfnDT!To(E}6%CxMh}@*Dc?}qdOkG6G__sre zNwpODRmhnIcocWBX_08l(wH;QNQ%qwY&3Ny+RACXS=&T|UDd{Vhd4DAXwBl~{&ZjnYm40@(=p z-}`^>|GoeB{@?q5@Bh93|9vY*V7>_*0^v&qTG|AZn~of-?My&Fq>^EEK@uE`#DU{Y zr-9lEkYIK5L!e2Nq*%R^0(8X+oVTJU!fKU2u+uIEdgne!$JYmr&vJr%-faMy8UoIH zdVq9C%D}M#JK){QpxjEA!TGD3!1*I2;MiRT9BUp0YHSP6KhyxnOb*~Uu@&fhE09l; z5iTEXp!ps^VK;%D9tqGV`rtfdBTyvHUtIv;*FJz_WqEMy;0(&!EQBkM6ctuy>w$&~ zZNTcz^WgYC3s6~iN~|U*0)KYk@V9DC`s?%ADj@%p9mH6D&-s1RU!a(tztx0~wp%0lH@qD7gtpw@ni$+!W|$Rp9?(1<uPZp?`4ME`i{d91C>qAh_$8p#1 za18jhasYfH0BB?~&;UMMeH;emDX9i3yAhPb5!b)GWc&cW(eqX9RxsJp{)EHQ@NXC-BF?f{Pn4a9z)6K{_D|pzg0gKGanp-G(f1 z9`O`smq>@z_n|=J4+8Cr13G61^s)rdL$p9GKY;vm6{xYA)CKBYJP@dPF|IzfLHbV= zK$Ap)HW>lEL=Mzph8C+aoj?^*fEMA(!|9GoX9M&k6zE$zkYD*rpkI7}su}`S)CbzK z6;}>Zpf3b~UYrB@N683t&GV2KcK8aNcVM9G{j2%7OcN3}2kT zMx09G;tDN}BU^w@B;d*;2>f3N0$NuOG)E8Q6QKplbLSm69wPx-`v_N#9-tM)z>aMg z_~kwd&a-X;_Q^Tmc%LgscNACd)^?yGhe5hV+_!YR5&`L#*uiwZ_1>lc27APMPP~uyFi#mW~#1-J*KNZ)1Y;p70RiJ~oetwc2q_@2R?D-CY8%M4?U>tCBj|Dfo-_*)&je!eq8!9KnHyR=P?H9gKh)$ zPzUD&2EcIzu71WDf$pOP_6hreMsEQsYYtSJ73lL}kbciW;718JJ{VNNdFEn}PAn1} zpL7MsR+=E)23&to(#G|>Xu!|XgX@_i1G-`f&OfZg^*>{vbSJ@idOM(Gxbl18#@Q?` z-*dQef?L)>)NuMb0w^7>J>@1rLW}(%0jUMTExOMXhSzNh8fS<#- z-v{EG19rV0pgieTpgihlfs)|H<#ZUxx5^6~H)aAIiUb;g8zsvpp|aFCBN5@d994^T2( zd`ZE@%g={^9>ZPVGhF+xNCKtV4b&J{FDxEFH{nzV7ay4;fuWZ&xZc;fxbwJq%0zd=IfF5uIYGRE$ zzW}te7U+W*;P;^c$k)yX9KSmW_+csBe2;6l+Bk4N#TgvG3IxX;_ki}j2Io_^0L{YL zBiaDJfSb3gSaETQ2lR6;+;ul@2j^*U{UY293^bK_5I9AbfyR6S`ehhsW)lxz{`?w3Go`+-WmN6#_O(f`4$r?gEi zHQ;a*5`};U1i%p39|`-_d2fGj2qumh!~NkJzA%`(x4);hyC+Oc+s_r|ZV}{<^7ey? z`Jt(Zq^P)vt9yv2o2a|5FGTXUMi^=aM|y|q!F(}UVXS0-Wd%dIVRH~e=OCtmfQKMq z2&9-M+%Eu%5_JW~5T)N*YQa39LB1$kq`NOBM_rgd3;{*K5u#9JKqy4yxAuNeH!p90 zn5Y}v-^1Gzvh%kFNEix*21rpqXaGd;x5-0buHSsGUk%d#Tn*Cyu^OcRwi=}Wt{SBO zt{PQV9Z8gaJT{XzySA)R1iLsaL1`WUtRql0@;{2_Daeg2bF0p8+pD*E` z72Ng<;H(8jLB-I~16?2wIE05_5?6Gb`F4qd1XFqX!d;=ho=`uSAJiY}iEe9%_+KPO zHz&H{kr0``w-H6akd_cmf`-1ZV3@D?Iu{5M{E-`p-?@?aBR7)2b0Z1dND@qCjI9HA zF?Dnm___LqiD+7SK@kDi;4J!ED^2u>_%<*w77D+$aD)4aY6p0`iJJQRqOB}ph+uCw z7!qr3{72RSzR)oAr6IyXpa_^K42~4hcK5~%WF!g(^%D(3dHZ^!!mtjN|IHz~lOn+N z?D|Is7+ZsXv_(Sw0({Xmf*w)6FlZ3U3+qkoKYGLXF#L}`{9&jNIASk4f?(b0{zrFk zw;EBwJ1tHwLpx7oyuLt~ac4EJ^ zvw($oBVg{%n%~Cx?+f_N#May29XziU5 z5P^wRxK_~oM_V0lUl`tX|GpRF5}Tl-k{${b@RwFWelUc$8}je!70?^jAL@(zeyjRNKCBQ>e|Wzf~^O|H*~P>RaT0s~!uOr#EIp{(A%eTW7O$ z^Mbht`Qjqlzxl9&`NI5QD8!#Gqkieb3ju?=f2)!|o&T(0p(tY*(#s41|En-=g$@_K z-}a+4{>(_*-xFh@?uMOtT(KKS=p`V!?*3+cee>dujy5Rt?ZJ=k{relVRSxv=^8Uk~ z{LR$g9Rc@tH!#6>@rUce?=HOkussjY!(W!)0?+ghyUaIJ%tow6kiWYx49~|OF1Wt= z2y*v^|FgBGFC68CXZ?p^;JbAYdbx+duKe(v{9&M9?F39?Fa(|-`M>jn9(;Jle;A9t z86%)bFAErE35OoJ|2!D9yb}0zyK$s%tZS|R&v5`dfQ(?C z=o{02FTt8Y_;+)Gfu*U*x7`QWKgRlyR=2{v&x z&vAax06hCY+z9?=Z)AyQ`-k`7I9uF?_uq9ZYkh5BaOaGd?;mcR{^Y_0J>G+1|GbJ{ zwSfJ*#izBOD+2n@_W;%=+E)L(P{aQJ`$i-9T>PJh*lK70ybi|xAAMKhkG>iF=a>bY z1R;F!JpA2a0Q-L{7!rkE{h{D)aC}dHxZ{uS$rk2n0SgR5`}_62#Xqzgp361c7=Jfq zu5pIv=MO(U__ZHQP+C2c|6P1s3sG=$j=jeQk8}UaPN$v4A5AW1zCq~Z zNHNPF&sCuVq$nyBwa(4fH>1#9vd*&7Vp?JT=oOP2?$asEG8cvb1s4gGmHy2{wZ@mW zsimcsp$NJPWu-+dy}Sbge!scB#)XF;+#QL+4DsIt0M0tAgQ*NDE+f_;CcnD9euk(Z*`i9Rt z0JAlOd!E=EhBG1jj3E+xcL(ZC{99EMevI)~$w77hSr!t1mxaWtEC@fzv)bl+tNvM; zB>%2VlB>!@__pHvZ6bDgg2covF&G@_3jQr5GVE-sm^J2g0|(ggp3q z`v-;oM<0He{jya)O#a$O05(cEBSXAVZeG|S;{G2?1tJu_55g1GuY6#hkYD|Pye6J6 z!jFfo^<@mje3B&U3iaQ+&I*Ts6%vKO{{0Hpn((vvYpt;p{I_DG7ur7PAiw&V1?)v& z{#X*M=rNx(=|M4TX9PN+WB$x!<2Po8c|03@e*M4K|6c!l{h#%k>0aRCa8NLqYq@NE zhRt=m8C(7`&1+%3JvAQ(5H#0QNej46GDm1JGA>m`svd4lkv9_VmVQ9vzJ2Jbl@Ze2 zbP9R+*?HCg0S#xtE3w6?TI^9@&Ec7PlPaeuVlx_~!+1IyUN%fLQl4qJ$HhT@K!E9- z^6&=DrfSv8v!Onn=cl&#i3!yw-}8Lp>GJN$&hg>feY&!NM~SMZ`2_p7r=L1$YhuVp zWM|RuaFh1)RE-k6zjQwLPxA=R>xNF&=G3i8 zZZkFRh~|5*!%&6Dmt68a)%_EP!4kbh(hrP>HxZv6XVl`^Cu{b=-x?i!mEzL%Kt* zMIT9dZgh_1~=k17meOdW%N*G%d%J*;s^d01#OODu%kZ?t4`yDmQc}gXD`!{=zAF^Q> z@4CrBdcGrV)w$;JmNy|fu$C}M9pRrpQH0_sz>@29r zy=GIze>UeM+@Iu%`c( z8XSAw77)QU;^zCHzk*bD+x40L*C!h9ebhN~O@=9R`?Bv*h7A`^d!3#SF(k4fU%6EK zW$&po!`H@Vr@82y+wz&tE~!_&>bGhUrnU*XcX#jW8brS;!*JeF<0nZa3{h8ej;EP! zd}XP6B;fhcJOrQB7rRUBkrvmeSi=2B}>8{Vf5lnK}x?J>V{##G~WMvLi-_czKDx=Js&TD89Wg4C$)qrXMP zeqWAL+r55I=Kp zJZZCa(#kab(FfrS&a{qE-pjs<(zhCxe30XBv;*77xJ0hK(H&GIdB!j7-+X;HV>PSd z$3q4zY3;Y`tv+lG6wJK=IZv@qzN9uvpju@2lw0)@W0s0o%XI$9;Jw14wlPmyILbC? zLU$;bXUCd(-wvlPuj2ez_vJQR(TPC_Qs#6|$FA5w=c7>=J7=0S6JNBoR+!16xetPrA}`yYjF{#MSSQ# z!+k1f$h}nQ6u;2TYB7oDj837&8Nrp5p1m9rN0rYakA$3`p59P;XdvriqmE(gy`h&Y zo77B33V22jdJiP-eA8h@H37MvA3Z`k^E9}nf|-pKxd6;6pA{F zjkm2pN<^P{;M9~ENh{sVyV{A19J(92!=G#b8R9BWM z#8u^Ta5i@2t=}VxYx+q?dNbm7^hg-qS7uX`t(<&uMkj^DdO#l46u)Tk(Qfp#y4*O=t;D#h*{;n#@3>Tlx)j_= zb)<|~K2T+7%|5kV^foe#h;w^(;bA5scUqcBc#9AdydR5npU~3A|`j5tcF{q{mYy zTqoZkx^;H`<%x%bZ-*hnyRy4y`edXJ3TaYYDLZ%VJ+T4m))Py zolbj9>wo1ksi(K42WjGlMAsXc`iEjh&rF!;hlUw;$E>ue$!;{z zYn*#gQ|Ebm>}p~a_p>T<7O z8hl?-OEQCeZTD`=%L^_=h-k74cG1gsXX3kFw#D8Hu9Jc08#BN9G*xacy;Y>;s2o+8 zhDDKs=I)eZPl^L)>`4-OJ*TJ3J!wx4Mb~V5I zIIW$h=AFfku^X?RcDBB~%T!L{HE`H6;j-@{5)aRpuMaUgV$Qj)WWVjS0(@0 zlk3oRaVAa4mT4sL&V#M-yL$b08_>x2?0%rJ^^ki9D#3L*vA%Wp(STe5C1gbGQ(GrD9jTE#AwA~?>S zBOcDOak*}>n#3AGtJq0@^k8M1%R?QbQg@N;$(L|>r7Qqgl2jU% zr`Hf4-4aDtUw5xQ^NMQA#cO=h_sm3A$qLN;Wt^~{ps%7-f_&Ep?afBll#DEUcbQiklj z%SvK`ART+4&A7W5O>ad^&UD1EmZ{%rYHuOc z{1|ft*;sqICd2&Q3v*wM7g(1KuW9m0M7{}1fE`F6V|+LWulRZ+sxiRXN@MDIsbHVh zLtpo*JR8?M6V-+O3+%pj5_?@JdyuoLOnIn+rJBqkxVP7-T#bIzhHH$Qi*7u(4&fou zcU66!cGuqYR7N{PU_%g}fR67?6A7+;sM%vv-Dg<$n#&NQs&75kHc(F>ojr|`lYB>2 z^DOJer?@LVT`nKq&+*F5a8imX`M^h}xeIRzq^Zm%Cx%X5O<_1I6{09UFk4U_V`Z2a zV5270{HolX>vGf>(`^XKq{z8FP*%K4v4bJN@W`jq+_ix{DQaiDD%} zB5g}nHZ9p_y#`xy`)>Omxfofj+fj8U=jd^nqn9idA`V7MG$~gJ+)B(6K2>k;pyj>yzCCytbb-uK~2M6cgAFY5cW>3YwPCx4i2aedqA zPt9+h-^A=^nIC0snQLLq>9LbKZ?{oAUMeBttCs?~&!nvbWS7hIY zYi&;wajMdB+x;{&vc9jT)p79X%IuKKg7y>Eox3KU9M5?Xe&kStW?Q3grfC5i)!d^1 z?eOR=yT{qZv`)Uv7_dUl`iedJSQ$Vx!06SWsFK=d_a?r;1$jlb>4;j9a=+Hlm+DP_wl_5Iiln0>Wm5$ zPOY5kMCQ;(X*ZwD&@E6KaI{B<`BIqdwk1l=P&D(&NZQ|`vyHpW{laFsmuf5u<DGx=~daOg#H!qgK|lIn3Q4_46K+SKGP;<=+ZujuG88Lfj+EPeINS zWp#Q!m=!x|VUaty&z0kOa6vs2B73gQz`Utyn@6Am$IFr{&tKV!IGK-p;4I3ZH0=1K zs*sqXJAY;FsR(1+fv*=L2d!!+GNk-VlthhU(rlXU2wHvqIK;?xc*i^1`;Pa8b%>WF z3Ny;!I;v3v9Ep~$atupd@>B8!UywJdgI7L7A)HKiCr5hn2Zd6`7sx(;H1(D)%r%gF zDh}-~act_T2)jh#a?H(izC(B@R)AKyXA{Gj`sHMk=!-$Y4veq2FH1fye&qC4PEdKQ zbwtEj`pC{q*$TSF7T5j4CNUzrNKW^1mb7l)f6i>@zQdVr){9&>luxXr5n0x5pC0FX zc9y9+Zr1xcUp%=luK`>qqILSD<2JA7TA7{1zdDzADdXl{kLqXF`E{K)`F@F*v$He% z)j9UVCp|tf-Kp30v9Z)k6+)Cw?^#aC^_NgP#bQ<$5K)#LTTrunxpJ|XoY#Va^wlQO z_#@l&XFmnU++#o7wtq(cvVfEH;rE{cH!$~+aL@Z6HWrBxDMpceP84GkAd}i`FnM6I zmp6lMM(Q2p#k~=7^YA&x{%ZQh0M)Zza`&!z*o#0a^tE~i4Q4a>qX!I^l$Qiu7!>AS zpFGXMyuF-vgY~a>en|hn-DvvX>wmBRz5efdVZMBk1O3)HcAcR@Q9>)(nJ298v-xL0 zH)7d|YLO3HePuOk;kG=aSL`M6*>fKR-37*Y(_#u*noh<*@FkGKln^4E-=8>|Vg44H ziU7Ao^V5A4SVqd0$Xa0TqJFUU>Jt+CwB7jm=IBXJ^MB>kD74@wS6N9I!)I5cCs;3$ zk851%dLFe8?E#-X;D3x@5VAC?-hMav4etvle0FGO{TG{SVMdLdl(XK&Q7!oFS%mYM z7X~W^btX5PDGMv3@R{XDvccub>v|=<=Zl&L&h_InqxhxaBW5=>8~2?ox4dG8{jyv7 z3y|;ivzB96vk?_ReuIn}qD@ZNe!~W8NZ~ADUb##SNeLv(WqfELD%6BI5=8?+iV)_L zMKtS{Z)XL~y5;N6+yGIDC(J3?Y1b{E4i7B^p--4M@1y|Hn4;_RufiN$9 zMz?PH#$VAvBGBa{CcvHK=piLH0dvzTAx)h9WK?c!#=MQ$GYHdnXg`vC_q~>5nCJ!w zL9R$n3K?=I%$fJ17aSj{2y+-6_PYrQaVa#%e7iR>0e%3@Lvo3*Y)IAK?bei}_Q3@* z9+W{WHs0K6CHZcTmSfnx>(p3g z7clRf5pYnql%ct>3;FRX#`bn3F_vkm^c4*Icxly1u}098@tEL`4orT6!XCug0yX59+4KG@;maV*fuqv)nvdl$SY`4~` zAa6K6xRCr1pWTXZ9hhl-sGWd(G#p~8%#F`PPYRmW`LJnLg^V%ApTCR!9$J~rWZ&Cm zEypkyyfM3$XX%nbPNHj`n2^051>{x}C6?8leRfZLpRIPB4mpLV)KukwZL-sj!Tr!Gickp{zm8#$nEhXV;qWui4jrl+mBJUZZ)Khv!H696@+J|J)}kxFrglW`T^&oTBl9jE`EH)k6ja+m;R{hwH(80c7yte z;({Ea+dr(4BD`Wib}#)C*7oF&IJcH#7-r0o5aP@vkeZiZ-sD!^`H11o1>G!lN2pub z9b)|cb7=4~!>htY-SgJzEODJ(CD?r9NWS}B%Q37b3iy@3*0^hy=!AZ3s-TX8I(;g> z-*f7>C$>Ce*EuaRLadNvhy9X&^P4o^>tZd(uo^iKf3nU=TkQQru6JJFTkYk+{n#I{ zG1vGpe%EpggH%HZ^LaGCN<^5~qQ`{ZI`){~N*P_peh?jB?ytkoP+`XfF(Lkl3R1J3 z9Lp4E_TBHP^)RUP-we+w9uB;M}OZtzKb>3@{a!~?^=#w5LRNs zd>LI!Ti3B?MVI*^DPi72v1*R~iBF=r9vNXSO1o+f{)wy5LTcK;nsl>Rq%VRsQ|+wu z`4Ru4Dh!~lXutbj%P|ZyhY}OwR%qXq!1rydQ;^GDbNX=uS+W^a0oWhbF!%Y<$JTNT zgKQ-t%#+dm+kxzR-aqkibbRIn@ww1QtCnk*vf2Tsgh-_+YgTOD;y-+^B&*u&Cw>ymH?PAlqB)9;Fc+m-wRZW5D^aa#m!G%+dh9oXvA^5vQd7t^ zChe37$-E+UE14$Xzv;Q)w0cfVYu*q^^3 z$feO-bRF(Wx_#Ec%@2O`&9%I3Cp|>9A9rt^_@>oo{IXHv z(5VtqnUJ%!Yo~>nDucPIQhdUpUHFzs)9#>N!R-nUFm>zg_QUsDj$zHIggN&- zWYw5i%ZIdxul0Exa4oR4w|o1zd6UyymKAyK)q@&;3pou zdLQK{o{HwVgzGPdpLSJD|B0*6u9|ay;zqRV)~|UM?YjMkY8=f&3G+4wbmlwO;c;~9 z`VYH@=8lB9gfRWO{ck#$9y|7l3Gkv`dWf?qU^htjUYq2c*FHM+C7ZYX6ZQvg?eWIm zT8?4OkwgSJLod;)vaRJ^ThL>RnXrAVIC{?bOiYl=i;zQ_j}Z3%<_ek@6XvH~(PPh* zFo#}4-yhukv6t`@PosepBEUUxnea8{Q}kaAE0wE-Q%ELPFm<;nfbP5RwH(8mMVCbJw<%Jt8kcK% zSR(rVah>`NdxW;%K)C-%yr6=-fe^OmnWDn3)0qfyMrz2@S7ccB(>_HQ2_$9#*iBwJ zCd8qouV2_8w!1()5DvkeyeFX?pYf;cxL&DJYE(bS)$m$NcGc(W=P7-{KALrHWeifZ_*by~xxH^E)KG$m*A^TOH?d?vH6DqVdsMK5ko@OjaVn(g?^Nw~GzIQ5ZM zv0d%;s%4Mc`0TQbJEb5uv3hb<`fT$}7CU^#`@By1Gaa|dIqAKXl0*B0@Yz7+PD}qG z5uJv<+KiGQGh zXpn>X*yPc?L=&0BY(j$$juRgEU!j{<@?>=-5 z$1u#FMHA#nWDu{cAA68%`8WqPWa~39{=Cf&(g^tN)y*l8m$9naf&KAXNg4|5-~RuK zW7yqy;4?Lz{f2vpGOhG8x9>C{+w^t`za7?IFXmb=KL`8Ny}UJt+Q~K!^SB=k;J#shqBnuiP z?l`qKJA`aIJ}XF0I2d>Xs(*OOm*4H{eldI&5_x`SbE=f)frUf!jJ!R8_{{SI)Kz4w zzTtsVa##3Ss#RlRG@!1nIYCb=Q*ktNp@^jopKUac6=ThrGCJy9N`p`y5x{3Lxn%aG z5BzoW7MTa#)EzG3Gh0!04|RE3jkKT{<@B^2UHD9PUlM&L@dwk=L%lXyRQ$vEY+I9+ zSSae2afbf@RU0w?aeOu{NPm{SazOjs=}lrzO%6Qx>`6(E`mC>;_VKs-J6fxx81Y%e zeiL=04=ToS@;x`tb*}!5tKLkqL&C&IJKg>Ob8w_!3_jBsjLK4FVO&>?H?J^W7N{p%Jc1f$TaM9 z8lS=HS&xr?fa@JEt4!NBJVt}hx;Cm59nU1vK12KUo@nMgJ3bq_O}Uw6}hUPnRJ&*AP zEYr#8mW}Kq+NX`ris;_9^L4W8#B<6BBb*a@@!8H!TY7SHIkjVr8Jl85pKZlwnho9` zWmlvPT^_jm7@sjR zZ0R&f6i_=+cz$c3-K$mO)`E0iyco~!?s`eI3)!0>-4|k7?#LM@yob{iCH~Kz%IS6=97H8kj81{ z)`#1v6W`|`d75#q)LllUczVX!E!>QcYREcc+x#@)@51M@2_kRJ;@Na zU%f6VV@$R*&IO-!+|G#GDkx%FNu4gMyKqGapXvCqm6Y4IsHIzNA>!pXZ^dVNLPA63 z{Wna~DMbc%Jaj*c&l1`fkUkoF^)8+%oAK+u#(>XU$gRwI?p5jMUxt2e`*>vvpOqSD z83y~L>Sx(-wYjxboy2Fksnrd9yk+`vr^rM*X?Lu8UZ*of<=wOR0>d2pzLN@mn;llw zOERV4ZGlrzcf1(ay~%it(C703T(LMTD~ClrKe1ZzcB24hP(`D?yps zuRc<~E@?wKp`)pg%vA1+72>g{SPDySWPSzd1|*x)>=~h05|HQ!fs!Lc2?2~HdO1YJ zeO?l=988%OBzlvVkPNxoqx+k-kx+Gj2RRYa%X>&hNvNtiClVm|NwuNY0bcP3Br~Zv zsAhWQBvGGF^s$%UBNUR=s+Y?}+=U3mul+N|)@WA0l>X4Hh=h{$s)l!|psi#Ym) z3X;9)OCbz)TbHWdV$r~UZhHsGq$q@G;+AivQstU=i+!>Hd(AH7Oomf-(*@%{bUz=O zFk^sZ@v0IpF!d~?qUWq1SIYy?AX$eu3bBkjmw05Nz+1a}L`O)5$jO~;o1G%sYWIS) zcP;}E>wnXGp?WXp2$yW|*712nO+m8WxS)8}uraB?OFOC6Lp?l5_G?sJ6|fvJ*%w{{>Nn;mR>b-)wStv6 zzc;@1;{o#Vlz{^(iI%xs#|ITe`nN#Ic&0b2R zKj}~IIg0?~veFm7PJ32T#L6OW{jOh?0OnVIKTBEhy^&-qHqgJiv@rl;E3G^v>UHsAeK&8p73wg6?-q|_ zSh{}|NgxOrB;$*!f@FDo^WE5+pQUOw2eJsl;cTjakXCathRd!x0rWRJ&IfC zK(d1dqkRQND2cFSvy1cbKtNte#10;c>?aqGsbsE$3HRNEWEvNies!o?LiyQ}cw;A5 zfcqk0fw%sNOhc+lgFOGoyH6#MjMwP98$C^#WMbAC(V|cD>&^FdBTb9vF-D&ng6_Q;mPmhHd4@&w8%am1aQ3B=(lP4-DNE)gl&8(Cs zloR)W<59)3)vRUpN=D5tGOgvZ1AHlr4O$3VE=x3y+B8lHngi(bs*2w`j$8!O(O>YJ z4n`tCGPcCLj>2Ot@u;R8rQhVPfLv_GCP!XY*d(2GJDHMPGk^-oV0)s&;4X%QN`|%= z+cq`uTJ8_4cqMMw&kPgAb>30hD*S9XuR`N&|2R$L2et;Jz$NR~7zUqmDMLabb|9u?D44dB=292)V|HN4Vg zg%vi+DQkf7AW(?&Id__0vJHW*CFW@029oukrg2Vai-`oxC|cfLivsGNRy77irTs{e z04`mUaRj#zz;oEwkK`Vk5v#LUPS+J2B!*-PPViw7b2SQKn9+xQY!M!iY)JIPr8z=e zG+-M26k13gkmL9f{l^V?*5oU2p=iAhO9AzFB$>bBs8u}vLWa`=Zr>Tm);UJkG?YW~o z*S<)gXe;rjd0<_B(WN{Mr}9-HA+VKno}0}Xl7+Z)<%=&O$QAVXD$eXV0P>h-2Q!*D z7M@5M(b<&y#1f#_Ajwuo%83jon{=1e%{n;`uxl@lT2tX8mZ;oo&o{af1MG{1RtwIs zw_+AA^bFSe*}@2jSsv_WHaRG2^7&0qUpEFM*+8<1TLMw^2r}6!e+{1Qj7U*PrrM%eORYs`AvCFMk&1YR- zgjXHlehKJ%L5!q?hDzj7nI$f*x)u=2>W zI`WZU@x%F0(*eHg#le9?SY9F9z^o;!8mSH#({bXymuv*hGEp41(KMUcfL@CKo$5uf z?36^_)`t)M58{A2#4oV(HB=^=Y(gb|SzPd_5+oy;!p`@-M-eM-l1O;-{D&JPdzKti z?xz_e{P}5iG@TDTpcZ~v!}S^&ds4Q|W(?MFiw;oly3poV^xu2QhgPKo!oTcZ2l!IB zYP*{(qHJ_{;_Xhk9U#YDv@nah3}{KUN|`yRAQJ)ht8(QDP`p6j5DX1Snf>1?XG3 zGrBDgAtXx#?1kbj4wnLYKBiG{>3p;15^3KHHI)qv0eiX+@_rUipSx@}KSBs&?^51AaI2J%(0O)`eR>IWMoWhrJ;Xk`Y(Ptmr>JiWZ-K zTVi^G1(=g~+u))Kc;JgBLph%*)RqF)1%^bICQf#>B?_@u$UIFr0drVGU1nmIIT4xI z(@hw5O-(>wHQ*lqh!Uexu+H6t#_iq>upVR`z5g9fS4uFVTWLt=;07@7PzV`Iv|m+{ z%Hhu-{!Vgp4ar_tyhc)Z{Gd=#qMTvM$N`8GX69GJnVI;qRnY7)gQdHGeAL&?H5o9Z zE939Ag{%6>0r309v@>>>j3i{K&zo$&lglRn{X^1@yN@R-E+TdOU${LOxS=6galY%u zAvA?Tc^#h_Md&3uB>Ta9ut4=yR;C7{F%vC_2av-JRF^cZ)hdNPzTOTozB1s3WIu~K zJm|g~OJ*fXrHt;u0{T)GpX85pO2ozdnpPD!7@+}Uj3NB$!mNluq+!GcMP`!^;7gd? z?{^K@)TJA*U6#n~V*vHFR|e)6DQbH8Isy1})ya2&K6OTg%^hm1pM-yDeD}k(JD?{y zzC^NUaknEAji$I%hwcjK4V#b-eo%9&5iK*4Vm0yfGK6G{v|Pq`;?&aqMhcG;rfh)R zXo$;IYS>IB92kuiwS)WuaGpfGwAJH1DKh0x(wt4EJOI7gpNZI!Z@HPpJDnZ%^PrRg zW3!;3=T(5kH>o13_>FaxM`cJBRfj(JEzL%{;DxNvwOvI&Bzr>j;vIddypaDFzhIcS z&`U_RA;qigjax4qYNVVR8o~$gmmugkls8K-LM^vR#1{rqGLUQrS20Ps9#yEMk{O#p zzra=>s><*3&(N-2#u3a#D)t~+ZF!wc({J^!Q+%24JAGYY=hY!dd z8;DsrUmV0G0;vsEn$HJ6L9!1rZYQ(Y%EFaD=BI7TFgzjIEFNCe{uc?^B-&)`D)K16 ze2l2Igu44wM<`5yGUKTHGoY4gJM~s_bW4^jR%Lb06ZqT)$#g?DJ8@k$Ws(nMD~r$g z$sn25z>hSB&qJZ^sA%*K|IR^-YwO?Fb!}2;P&IIW*{OP<(*AJ96{{ezi^2`s0PGjw zpz#!vG943-!A)pjQ^s!q5dwTPD5U>&%)fIG+uMWwx5u=RM}zta^Os$&h0m7u=Tb?( zgx3tdxgvmdK>cj3te_W!QZd)*RN3&cfZj&H9jX`~L7GsrfI&-X=?VwHw*@$mJCl`U zDoA!%DCZ6Vv5AEJC|%y@jYxZvP~v6zTY%qBSWXNEDwK;CZmHN#z`g?a3^Y8@!GF8- z-*FHt355E$*Ql$A2DO0n7fU!UV0ngHE*@OBv(fmbAqVo9crWP)9QjBjVyg-JhR z^W4`{y^L&u5i(_(^(g%Gmw_ z6{U2*dac|_XPSI$zC<!4y6pa^bl7JfLSRcdxw`P*U zM+fPj&h0LM-oQK0)?CrdL)ok#|CEjMmSaekN)*`2Q$QnQMALQJp|Mv$;TLZ#B)tDl?nY@36wQ}KGUcP%!U(xmqMIn3|a5SIlw1ON+LSr_|8N?O*}N25iLPyrAt6!5dk zssd{zs#~AcAscD~`sUm**|x9DmL;PPSCwyy2LbayJ;`4$zKsvdwz{S0=srsW%rV{w zM{e$tI*ZqNUclj9l>qkp#Dor$maa6(1)ohOdh?R2Lo#H!wctew8tqAjdE|^7cgg|kRWgz0uVL+33bp&vBzZ2mUm#iDiG)^&C6Qp{9@M-6 za}=PD9Px$=u>uiRJ`COmrCrSo@SJagw{Q&|WaRS7^smXDc>ro)S{sFJF2@V0S{!^Q z!wql1y5H}+iUX=VD~0^sU$28sE&+31k?UMInD;2M={z!=ax-CocrutM=0YI^CEbAF z$5`3@6VSKb&RP5d9Xu+U8PqfAC0GJjd+s?+>?ND_QTV*tp8E=J1~8_SW_W){aK?%R z2f9Cc89f2awG@g}%FH7JWXe+eBwk4c0%C@A{mlc9XO~QnTgkrMdVv8XW3MUWp<(|Z zS>(g$!0=ui(67;d4cpTTf29zhA!<}mRS39e>vkB+ZT_F71G4DO5fwOCAXy6@6~Sv> zfBAFMTu>pKOdxUyQWi?SQ0&~ccsiE}5`G?{s~_$Ky5D((SpdxULjYM_tsWqv&tWm z&5OY}>y?{{q>D`EX*PByL9$7biRvVjQo%}e=jH>4Ex@~)JVrl|vc7JX48a{9GR;Z? ztmPjwg!w)p{UlqVUT?iAJqhTA54^PV`K;6_Qu#F^9XB)-&^IIRASr9=vJovGNng)v zNd!EjfE!%*(})oH#_$p)gW!8W&sT$DbVRi|OEgy#cdnha5a5ICC%ATZgsq}+T>;$3 zd67aA|~SiR|2pU!qWofOXW}!dtU-uLz+q{I+h!YAqc|hH~m@{}^B(^w|;N z{))>55MNeI`?Vt*RAn0_@O+f2JOH^D^X1DntGLfnO>FORrDClTAepp#BE3l>g;+qJ z%zI;r%NKCd0gZ~OJ{?6y8t=IY*r1QpNJoc3)0c%mxIrKgJ_Jcy< zqd8l=l5XH0kZk{d{s#K*9K4b`4$k*HQ;=l{GWM9`n6FN@@+u_>V%3FeFJv+K;fW#RIUY`ktdL>rb!zUJz80= z&Tusc({l{ayB5?i`)(WrCz5H+hV?E5A0LvnChl(aC$}>L35c(4Q3i<^;R&)W-Wdd{8qWZ|$nV27#=~9~ z(y?|)JhriSfL`RXnIuf+#5>_oY@>kfT`NG(2ID+bTU+?6$VVK7D3on(GDw!pTVnkO&1V;}$#$xdm~nzxl#zDG>{!N-O!44b5PaphRN1?>fseTvfZSqt zvL2-L7y^_Z{D1x~=kMGk1_3I=?yvW; zAPQ(B((f@U_{wDbR>OM$zsG}k7YX!AgMZ6GY>VKpy@aqs28rF$2Ez62>@Us2&H?M+ ze9f#bvj(Vg(Oz`fNQ3o&JcNaYiEhYSCX?GsEp?F42#Ce?wc+hkc_oq+hW93OeH?&( z{KvqEXOhxJQmsn-wjvceR*;Oo%))CS?od3ALXx^JU%d>H;f$uvLG^7)XDCDWXJ zaujw;NpK_XSA`<0B{j#HQNY?`mo!E4)8DkB2}k_h0T|-jkgOIBIuwtCPO!>HMG$&} z6cBH{M711c>Ix)7=%Lui;Yk2<7PIFS71Rkha*Z~5?XEpFfShlg^$sOU?WbI8YelWA zA@3|Cd!Z{tcQG~~lhgNl;SP=j&}&#M*uV*pvldLdag0_$fAofAK_;jTT33q_HSZmK zk5f8sAsL?OisOLUsB{!1X0o}3DPV0b;o?+{H|mXay-(9Y>L?qaMjWP`8n8JDmdguc z%pKr?1Jrti8V_=UZ8akP_IFq2Z0k#q?7;nY)obLZP^RlpdfYcBKzuW1V*F`SN3M|9 z7*$qDu@8tTN$ZKd&(J(Xv-~@?LOMAB`vSgjmUkQ%2#Qp1Tn~-REduW6(Ku8sK5DdJ zkxrEMuWzP+9?Z=?3-8qll6ZEF;|l>NDO*U^9l_=I@q}F=j*RFfr2`=#-u!(Ilc;UF*Q=y2$>T&Hv6Z*g8X!Ig)HIO*3Pq$(>`@ zL0Sub2 zpe`5D1)uYo3_Cj6%0~O>K78~rt$iv;du}t^+edpKr6WY!mO}aV3Z$8(Kt;aRf${$d z^D$rP*EDzmY<-lU0P5plPdSYagf%I+D>fZ`Z&F#-#y|PgfIls$ml{nE)NN! zhF&wHk@s0TfHYVpo%jugGr5&SVvIQijR%nSzU?DgI^|3pzQ218qu5M1NK;Mv?u?#l zC{@p}?CM&cGyu}N$IeB0yO`kT5RKiNO6G3<>Hq07<%BmosFadov=sS^RJ;Ibx&QawENE7 znO+=!8VWetSv-EBDv)OA1f%*6$t!C}R1oRFG!zY_!D(&K5YKgBUasmlH2JcKfHZ6- zi6NJs3+$M0wNS;88Id4u?we9e#LrGY#sK5;y%BgmkXEO}M-|vIVN`-bByjbqydI=2 z@n-Ps^Y6cWK9t)rg-#Ix(vs+nrZB%o{*KPuEE`H#kp*d=qZ924{Ty)F7w?cd717B+ znmqG!S=cpltK<*zA>BA3#vskFhq^daIS(6Foo@+lbv6m4DRZKaGkc7NPzjO|aYd_{ zgS2YGP8j{LOqMko(M0LsAA}$+o1prk-CpYY%P-8=#s_ZMAZM)=JjZ8cATu_WL~gA0+l zFYo#t*!HvkXXIHw3SNhvs~!}y$|#Vg)s2hvPSdE7%sWlM#e@|Gq*d%rU!yb%rE9+@ z7Iw`@0-p;hIfa?6wC4=-iW)~@i?atvBTVXGefPAl^{~yFC*!=q5!~1eYpc{CxMpi< zRxEY9*aVM{dvmv`rQHG&kpaz@)Xm8X;BncEEFhIl67|}UPQOs;z6GQSbZg3utcW~i zuv5z=zCr@W4X;(BspYeyF1!`VY~7RJVj%5lrn?Z9Rk7o%da8)Dd970LnATBJf{M5` zI5w~Axz!D=10R>t?_&FBIl=Ly60@;hO(RItIesetiV%IvzT zlo0SFbY)X(5ddi}^U^D8C34ESw|iM`A3RAx8vXc+dXR%;(|zW7H>P7`A4rpiLa!Jm zk*bM-T}<=wiqr#XT>}eK59OUQb2C|^;d*@FSS&+aWWX^DWn4nU;K~#0_8O$=O=F*t zM&NZKDwtuVt>g59v^SnnnD|}w+9=IA?d{i2rXY=>Ci2MOj+;kvUW0bX?YPU zmRWHAlntSov}z~Tm>}&uQ@bpzN;&0vdN}V+<@O$=6;faA+SgqAu`O--i}6Xtfa84t zW4-+BF%)+0kTTTvDji5$@LDBtmkKV4*XP7_Y_CcIX}KZe<5Ft+F0i0MT7Z5Hlnlp{)&Gp;V`*Rlf4ztf@*@TxR_sp z^GAZr1YDxR{&-tK;?d!bK1hp7&VFW%^L%u>r;Da(rkW9?C6bXe1U&I2DlmT!$Gs4+ z0@4UD=BrXTR`?1P>D5U{qF;eDN-|`^`RP2~PfR*hQXO{S{2Lc~tqN^^y=4%bPUuh~ z1q;#?En$$v2Q1vk6b9bBdZKg-(q@OVi6yneB+97ZX+z+w^FSJcGE^!0_F~n&R=SII zkl+MJdnYRFR`*ct9FJr=am6fk3DN?n^GL<$jdeV-JX){q1pPo7Zt<5%L`1ZQ(*7d! zwL+mm@G(xL-5riuZOevDk|`AYz}M@5s~J65gdR4xLMQi1k_|l0U&!{~tzhz>gqBLG zM@AxnbFu{wKkJdrYEFu(VyFGZyQX?9Z|=lQ)8$6xJylnkc3iv!1|d{v`kkyz1C z*2&)+yJEQDIgg$@^Uk|4JB@4RqKKLf6YxDqpq8t6>o=7sLD$}br2!sW{#ffs>`w$5 zbaH*gYka9(|#RCCfp-G?zG$GgVEISbMdLn@Fr zizeU2_tN|LQXYVxIdVs%qdn(tpunD|7(t848F;*@@DzJ1HA6E6c}!sGSFeKOno_Rl z8|y;0>}jNM{B~Y`kQQv!bLxK2M?;(nC#$4}4IUrzFa2p>2FXAPyiv-e37d5XX&+TL zQY)0Se8W-~<)|>#!1uY!JGwsy-IaE3JHV^^+NcejqcUZz^b`-tYn~9x$~$E|g0#ii z%}*I>FO}aP#7sBghNyt+4V>rninF9d-cL7Dd&lL$W5zI_#EHCJwY0KE066j?qz4fG~+&+ZXR$= zbBX6u`wXAaMboj7egzCO0IJ33X+Vw&w0IY<`hsti$>BA;2RGoKrfN{Y%aw668N(Gnv9D zVm$$AYL^O#QHOC{uFDIUp>7`*K^oP@TC$G2VwsF^|u0DxenhMgLVUIIKj`-iib0<_YS2;aYXX;0B#vvzwjt|Uq3BoO=2kh zdXxSXTw6GtNiwhB3;ep7TBqQHvfu(~sCW|_76?@=tb~*Pzda74!M>Y)3EdU_)P#)E z%hnr}7(Aw5AcWtyY?v5$!{(MT$mYX=v{E4?`B{(FAb0JrA7_5eg6mFW%Ib-4PQRDx z+tMuhtDbCtwAXTz`4x-G%lfZhA6za6vxBsRI1aZj8gB(mKN;<&Ut@stW#K(^miUCw zB`TT?_b#jp=9etcs0Qy@bQ0BNxL0%w!SYzHWxU6T7>mwy7sI;`dCWL+u(^DDpDS0v0VHK6u`G`Nnd zx(RgYVLIhJVIM{b9gudEM5J)icH&m^3SnJcwO9tEHOQe^+PShloGS=QVDlP)uOpIn zzVGYSUeAd&4)uKm88}XEi6T1v?mcdp7I9v>vCHlR=e*B1sEO%zdT;i8+8_CnRKdqc zJ!`(syl1aL)D$WrC7uK4JdFDgJU_pZRcf5_yJte+d~`VSh4}ZV`Ojj%3~kg|9Pn6p zS95=3S|chni-e+q)HVj5YaCe%8&+PPE#AFC_sip6k^^b;f&LnRS^h2B%R&7za-Zk9zU{9!8}x%c0mqFa98?up)AKmdj zy5=ECg6DZus1f{hALnea>88Wtl(f&mzNQ*mQGRPtbh9*$qDP}Bur!+XbJpEpX78^bhTyG!LbhKh#%No)#H{kO##=iTV+aZumo=F>NT2`6?_VJ)=5lXSR6Z+@VRyDbE;JJ$e zTS}(=ieg%2qJ5B2!vgquKzEfpG&bZN6f?s4CR65v^Do~b#Q+T_*8a4JA#)Gio(5>2 zk)gf4wS}>vqlKLO z$NB%_^MCiq{}XrOkHsI0KNf#1{#g96_+#=P!7p!AzgK_pej?f0wS7yRn55)ov)9gV)}D|N^bS-+4E-eGvps9J<0_( zUrs}z<0f^MO4o~|;cbt7y&sg3_`Cb)WN0jyPY_q;61II@kGRjT#d;DC1l&zbDLg4wzo>E>JsnR;(m8?3uef!r+_$Y?*kvaD93{n<+K0G*IMAC% zh$oNOBA6HF3&s!CJI8TNe|(8M_98rwiQ{wBcLu}@B{Wv!xBR{l@BJBut>0w-hZ zF&y_biM?OV4e?xWKRq~R?Ccys#bhfTAEpFjI28)t)bM02xO_nzwGf-tOFz4p7y3a} z)#TS;93Dxv@n6|*y+0j3@hdCgE*lL|G?&vQ8RYv$br;iCJVMy_+?=ctoj{)4r$fvXh8|x@aJLD7i+gc*SCZ^U(HepGexPAvNY^T zFI%dvNVj zo;#j^J76!0|Dxs2t*&c6|ChH?6E5RKK&dU_jde$~SyN^x#udtI4zcn8Jg7ljJCOs4 zLRW2nr*f)P2RhlQkjIb49&hidyYKl5hh=4DJc*?~K8}keFbW~CUCQjH)e+w4-RK^c z&f30rApTHY`L*70WdWO}V~^DbR`qBkPC8m2sjn$tMW)fMjNZW-F~uZ!e?LEreZs!$ z+2xmI>>?kArS>LftNP69z)J2F9Oc*0?7P&>2H5>9o5SpZ3qBxaF0J6n`zS zd82ZEiJ;udLw)+;;oEunt3~NuSFPumBblAWEH4mF>Tr@V+^|i{ws>^Cql;>?N9HMJ5BsJ5 zGH%so)+k6FR~7pR3e6W)33F03{V?@sUzsA}$0zu~e4H^#TNQ$%P>K%SObfLRpG=9x zIjCpPw^iNRpSMV#ijQtfZBlIcXp3_N)V<)OV^Je|wX3;1Y{Fb0V?^?^978A(m8Q)@ zUj>>uiEJPJ5F3$BZNj1iwP1Lr+y3mpcgR^(rNSdsKL^*Y_PLu!nNY_If7Xf&4f-It zXkA@ok`&x;)6Bx@Y$aUoM0VwlV&EEYV@a`9{9C1Ij3D|3uaKpwqO zfv!a_$&-S!{k9)%-looPA9?4V_@A?vp}E@^y`0k)^&Mk=i0J;+j0B~jj>kwx!)I9| z+5~g2d%hwsdXc&$ol`c+AXO|>?l|Dgr7?hag!-)+#y(0#9G)LW5G7;&>&+OtMgAV8 zjR27?rm94_>^_qc^PwuE^Oujej?Z6*{j`7QvtaC6)gU;_j2!v(eMk@7+^3c57lu46 z1BbDa!v5!SVK0y4PSM~=#5zgd=Ae>ur}v{|uXB|>rz@38!X%z9d{7U~%y@o5g}F9z ztecHWJV)A5MnUFM=2A!weZYTM^(p1Mi~!>b`|7=EOF(h%!jHbR;ayDU-z!B?jhvGO z*W6vW%k#Hfo64PLPvXNj)O2u$lpd`t-%ag^SZ?CDBgqN=x>EZ6vK4`w?GdJJS4noF z6ADq=()ki5HZ?eECpa|wW(1*070-GW+aXTOgo=RUshaWxD)Ce2P5xFI3?c#7`w@fX zkb_HeI86)fehlqrt>s*YcMLz8Gy8tkegDD`j3{WL0NY*4J-=~lFJzhp7o;wG7}(tt zcks!{utS!2%SpIPr#vQoi^F$aIqB)4SeA9NcAv)d8^c@*N0xJsT8e&Sxc0#X1UrtK z_S?v3Okzf1xMgs$)7M>V6}N^dG@iu3yBwT{}^^DsM0=)#e zdcI}Z*G6mj6TRQzxq_sRi-Jf^RPtZ-37*&-e5ZOR!{k_vpXFD?ZE97G5I`t&n@@Y8 zitH|EG*MEN@kO&dAji&Ae)7XBs&PJDJ|!rzY_%5$3ZNq(P&wr7~ z+Y&9)avY+{PNY1qIT@iIb?3Zt;Q#A6S?|WcoCN8Pf&iBQ*B}|F2 z7MYh~yK>*OJ)5IU4wJ+E2dj{tq%%Y`L*FWDCVsB#`$TK@&F05j%6i(@FVz#?)Tc`? zTsC7}K3uV^Cr9DFI`H^S=d1-4^Lu^LK~G%rXZE$0&1Xt3tstd^vn`vH?oJF{o=Lp) z3S@_i%vJJDOuADJb8~F!d^`^r&O<5mjkOlfs4rOIjX(QHnNmn)s?}VnCgLOdZ8yo^ zpAx+0+d?A?p*@cH5Q$gRZ)|6M9Lfz%xc!P?SkHJ7u4;0OR0x_Qjzs_LQ?B0$s)K9Y z`|)p_2M2T|k9z@39=16e;z&E)j(#d@FpKZG#<3H&j};jw-f0od9J}-TVEmrY=Aj_X zLZcLVx7&mpbB*OBXv3W;R3`Q80BrpR-Vp^$__cqnUMDQGq)v^nM`?Yeuy?+8)Ju_KkI5QehUc(wAhq}XJ{s3p3XLWU+$kWr0HL^3$Iu4~@-Z30BuG>AwMurf-`vW^ID>3Vo zGph(CB1+53luEn}zsBylK+d}39D7v1Zdli1qj}G=wQAC$L zN1|;SXfSDU^H1(OGBT@Nk0{_6cpr zRJ>hb1d22j+kxn?f{s}Ctq0#1W|!Mr&|VynWb_ZF9*a&TW=DE5bJ}GiaWYr$lJdSc zOZ^G|D)5WPVA3b$ib~dJ*zAH`_XPuE5!O3;1od>9H*wx@OU*9sk&MGS)%Wlm9%X&Y z5&8u62@7%`$0P~}jXioFo6n1%yje~Sr5V?5EY-3TXl$*r=vHlr@kg=Gd3zeTNj&B; zc*~cNfwwBkkS`adk0^lfA}WC1*5Rlv_+f^Mh<){qYlxK0Z~L`boz8}zP*y{p*c8db zj9s*uk*yI|-y(k|_l5I4_O@i3tC9UazTf-$n&+FTUrh}y8HFdB zE>XT23I8W&yVrhxj;YR6e)s--XW;`y6BC|~%|!gGp5yMy59X0!qnRx>B&75#TdVh( zJ^NIbV+iQ5+qvaxzx?H$7kmgx2ir*+gAWvw8v6GY+%3bX)k&p-ZwM97x_61S5|#yM z*4QoFPWs1)FgkN-SkXILwoMk;oTLi{XLo+dVjDo89Iv@mW`)c<(x7ZPsPRE3k&7`} z1s}<;*Y(=AQL;8yh{pavIibJC$N!PMp1W&C&fTH1F1!JpGVg`jGe)Lj{?2--J7c(nO9L@-rK`YX`Qf=Q{Kt0iPT_H0x6Sx29eQ+I%KXVAW`TZ^ zr5{`=zo*&miR$5fUb^uP5X-kSfx4shV(DJq^woU9C&F>mYiG`P+!MTU%L)(_oVb{%$IL=HF{!3jL`c2{(gU(K)AKI9zR(1eS63c%+)&h58vkoe#u7c`;HIlW2-Fd zuwwmmrb!?xK;VJRg zqA3bIf}N)<`B+=>Y3!D@CF1SXq!n0iT5#xgxobn1=1!aT>RGDOk2;#}b`^f-79H+{ zmlGit){3^mn;Hg_mz+bvDT`9?yZ-8@c=w2W*#0hQ|D4ITnXd9xH^-L`uCwR(#P$6~ z%|$*pbnKtIXX@JdvUqplJZp?8;(g)HViVHV!?w}4J6eZ%BWKx3-iPODDn*cI)Z^T$ z!#737-7U;T5BJ4Yrr1v=YW{MLF%-s-i&>w|I$5$xo&L5l8@y#!jDwQedXdfn|GQ#S zU9C!kf=*f5H@4PeDvN-e3APU-(>0bnj&I!8y!!!9-3^Vl^~Z5aHuXWKVKFL;fl49D z5R3GQ*^$`(hjSr&4*V*Cb5jS#wB$w7tYq~368k)cJiP8T;yozE9H_75bb0tT{wAo% z#LAz;i=QosP{cD0H$C#F1b*r+wZCm>{o2X;@T`$3q6Ov_El(F|j!v9tk50+AN2R$# zV2G3gAG=qs{r z?1p_nNqga1k@HSU%c@{GRwlxE&4eaY!`EW9|fTRmr$N6rS)co>}UCPjZAt`qd7Z6 ziEM=fqw_6+vxfcXYim4j7Vqr71tq~HXIPRJ#VX4xbhB5N?+6|W*d#FDV8?333Y^tdKU)>s zlESf?!0DZQdiA?pbpS?ZJ;UCffeH0mpRWS2+Es%hAlS<)IbcDnxC=#7CS< zX>b&!I1*GP6|i!s#GBdC<4P4JGUW<-F8qR-^Y~;!spWD+w?3i>2q7lar_ZWB*`EG_ z-zKdWD{SPOOXKzA#S2%BaL`?`!qbU@Fn6i#flhcJM_jxf zjH#BUv-pC2+Zn}4^Qi^0Z#t6S7T+0rH}{8)BvX187A)FcjgJ1MW#&lLsw0tB&m#+b zkA|n`~&KBW#WVGwe8|dep2%X`UfoCR%ma%gblX-(`}c~X9`v}cvjs{WBsmpTv=`w{em~hee|1#8 zPM6@4NktZH!r?w8DuKxQwzoE|zSmH1L+eQ>%R+`T4|0%C?8yU8Wcs&RYvn_p*LmcL zN5s@itG=U_Lb@&gF-b5{q*8>RG~wwNgQk8mD!ZuvGsNH?^SNjVJG&l7psejSz5Sa| zmp2MLeme!%s^lY0<}0#}aa#LMD8Z}^tKW~GjE2nY3e9OoFM7ym6@Pi^`0gSyogzNH zTU&?xyZDBgw=(A^1oxW7GPvEg;pEUq)ek>pee6nERNy8h^;1(rYyD?9$*E%v5*7H! zW2?Sw=JG5<#eTU&{!SY}pixP<>Vx~dNHpZ>2dzC@^RGXjohD(UMtMHz4$1m{8)2&WZAyt2Xl%@vcD{+(9hiaj!C?w<)d3_u26yTjfu(*8}xu3cG^(lNc9H z^MVO1{qGGZ*AUM{;kQCuT(X2P$<6u~*~^%4Z~38a0|S;>RT&YU-&3zgAF zO;ILSSK-8&;vQ{x5yuYJyZQ-4$=H)53Q6#qH_A{4(N7@}l17w3RiM5o?8}|WnaX?L zdsH6vuJxIdK_HcXOHSFn%YZLOA+u#?QM4LV_eQ6|2ZlaT`@XlLw~xC0_Ly-06)z*n zW`|+?e`cSWEH|I+z5nLi*{ZBo-$zE{l z0}cDN&4*7_Y)mY%v_MJzjB==p_d@XcTO7~Rs3wPVo(evU}Rj=pn^{u?Mwf2P6vKpp)y)R`B zPVKzp!(iHQb$KuM@|zYv9~-QatPV)$)Tr4y^{Gwv7Mt()*4e&0@n!SL!pWSl$xr}m GW0IMOw4Jg5 literal 0 HcmV?d00001 diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/Api/ManifestInfo.cs b/DeveLanCacheUI_Backend.EpicManifestParser/Api/ManifestInfo.cs new file mode 100644 index 0000000..8f3db2b --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/Api/ManifestInfo.cs @@ -0,0 +1,305 @@ +using DeveLanCacheUI_Backend.EpicManifestParser.UE; +using Flurl; +using System.Diagnostics.CodeAnalysis; +using System.Net.Http.Json; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.RegularExpressions; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.Api; +// ReSharper disable UseSymbolAlias + +///

+public class ManifestInfo +{ + /// + public required List Elements { get; set; } + + /// + /// Parses the UTF-8 encoded text representing a single JSON value into a . + /// + /// A representation of the JSON value. + /// JSON text to parse. + /// + /// The JSON is invalid, + /// is not compatible with the JSON, + /// or when there is remaining data in the buffer. + /// + public static ManifestInfo? Deserialize(ReadOnlySpan utf8Json) + => JsonSerializer.Deserialize(utf8Json, EpicManifestParserJsonContext.Default.ManifestInfo); + + /// + /// Reads the UTF-8 encoded text representing a single JSON value into a . + /// The Stream will be read to completion. + /// + /// A representation of the JSON value. + /// JSON data to parse. + /// + /// is . + /// + /// + /// The JSON is invalid, + /// is not compatible with the JSON, + /// or when there is remaining data in the Stream. + /// + public static ManifestInfo? Deserialize(Stream utf8Json) + => JsonSerializer.Deserialize(utf8Json, EpicManifestParserJsonContext.Default.ManifestInfo); + + /// + /// Reads the UTF-8 encoded text representing a single JSON value into a . + /// The Stream will be read to completion. + /// + /// A representation of the JSON value. + /// JSON data to parse. + /// + /// The that can be used to cancel the read operation. + /// + /// + /// is . + /// + /// + /// The JSON is invalid, + /// is not compatible with the JSON, + /// or when there is remaining data in the Stream. + /// + public static ValueTask DeserializeAsync(Stream utf8Json, CancellationToken cancellationToken = default) + => JsonSerializer.DeserializeAsync(utf8Json, EpicManifestParserJsonContext.Default.ManifestInfo, cancellationToken); + + /// + /// Reads the UTF-8 encoded text representing a single JSON value into a . + /// The Stream will be read to completion. + /// + /// A representation of the JSON value. + /// JSON file to parse. + /// + /// The JSON is invalid, + /// is not compatible with the JSON, + /// or when there is remaining data in the Stream. + /// + /// + public static ManifestInfo? DeserializeFile(string path) + { + using var fs = File.OpenRead(path); + return JsonSerializer.Deserialize(fs, EpicManifestParserJsonContext.Default.ManifestInfo); + } + + /// + /// Reads the UTF-8 encoded text representing a single JSON value into a . + /// The Stream will be read to completion. + /// + /// A representation of the JSON value. + /// JSON file to parse. + /// + /// The that can be used to cancel the read operation. + /// + /// + /// The JSON is invalid, + /// is not compatible with the JSON, + /// or when there is remaining data in the Stream. + /// + /// + public static ValueTask DeserializeFileAsync(string path, CancellationToken cancellationToken = default) + { + using var fs = File.OpenRead(path); + return JsonSerializer.DeserializeAsync(fs, EpicManifestParserJsonContext.Default.ManifestInfo, cancellationToken); + } + + /// Predicate to select the a single element in + /// Predicate to select the a single manifest in + /// + /// The that can be used to cancel the read operation. + /// + /// Builder for options for parsing and/or caching the manifest + /// + public Task<(FBuildPatchAppManifest Manifest, ManifestInfoElement InfoElement)> DownloadAndParseAsync( + Predicate? elementPredicate = null, Predicate? elementManifestPredicate = null, + CancellationToken cancellationToken = default, Action? optionsBuilder = null) + { + var options = new ManifestParseOptions(); + optionsBuilder?.Invoke(options); + return DownloadAndParseAsync(options, elementPredicate, elementManifestPredicate, cancellationToken); + } + + /// + /// Downloads and parses the manifest. + /// + /// Options for parsing and/or caching the manifest + /// Predicate to select the a single element in + /// Predicate to select the a single manifest in + /// + /// The that can be used to cancel the read operation. + /// + /// + /// The parsed manifest and the selected info element in a + /// + /// When a predicate fails. + /// When the manifest data fails to download. + public async Task<(FBuildPatchAppManifest Manifest, ManifestInfoElement InfoElement)> DownloadAndParseAsync( + ManifestParseOptions options, Predicate? elementPredicate = null, + Predicate? elementManifestPredicate = null, CancellationToken cancellationToken = default) + { + ManifestInfoElement element; + if (elementPredicate is null) + element = Elements[0]; + else + element = Elements.Find(elementPredicate) ?? throw new InvalidOperationException("Could not find ManifestInfoElement based on predicate"); + + ManifestInfoElementManifest elementManifest; + if (elementManifestPredicate is null) + elementManifest = element.Manifests[0]; + else + elementManifest = element.Manifests.Find(elementManifestPredicate) ?? throw new InvalidOperationException("Could not find ManifestInfoElement based on predicate"); + + string? cachePath = null; + + if (options.ManifestCacheDirectory is not null) + { + cachePath = Path.Join(options.ManifestCacheDirectory.AsSpan(), GetFileName(elementManifest.Uri)); + if (File.Exists(cachePath)) + { + var manifestBuffer = await File.ReadAllBytesAsync(cachePath, cancellationToken).ConfigureAwait(false); + var manifest = FBuildPatchAppManifest.Deserialize(manifestBuffer, options); + return (manifest, element); + } + + static ReadOnlySpan GetFileName(Uri uri) + { + var span = uri.OriginalString.AsSpan(); + return span[(span.LastIndexOf('/') + 1)..]; + } + } + + { + Uri manifestUri; + + if (elementManifest.QueryParams is { Count: not 0 }) + { + var url = new Url(elementManifest.Uri); + foreach (var queryParam in elementManifest.QueryParams) + { + url.AppendQueryParam(queryParam.Name, queryParam.Value, true, NullValueHandling.NameOnly); + } + manifestUri = url.ToUri(); + } + else + { + manifestUri = elementManifest.Uri; + } + + options.CreateDefaultClient(); + byte[] manifestBuffer; + + try + { + manifestBuffer = await options.Client!.GetByteArrayAsync(manifestUri, cancellationToken).ConfigureAwait(false); + } + catch (HttpRequestException httpEx) + { + httpEx.Data.Add("ManifestUri", manifestUri); + httpEx.Data.Add("ElementManifest", elementManifest); + httpEx.Data.Add("Element", element); + throw; + } + + var manifest = FBuildPatchAppManifest.Deserialize(manifestBuffer, options); + + if (cachePath is not null) + { + await File.WriteAllBytesAsync(cachePath, manifestBuffer, cancellationToken).ConfigureAwait(false); + } + + return (manifest, element); + } + } +} + +/// +public class ManifestInfoElement +{ + /// + public required string AppName { get; set; } + /// + public required string LabelName { get; set; } + /// + public required string BuildVersion { get; set; } + /// + public FSHAHash Hash { get; set; } + /// + public bool UseSignedUrl { get; set; } + /// + public Dictionary? Metadata { get; set; } + /// + public required List Manifests { get; set; } + + /// + public bool TryParseVersionAndCL([NotNullWhen(true)] out Version? version, out int cl) => + ManifestExtensions.TryParseVersionAndCL(BuildVersion, out version, out cl); +} + +/// +public class ManifestInfoElementManifest +{ + /// + public required Uri Uri { get; set; } + /// + public List? QueryParams { get; set; } +} + +/// +public class ManifestInfoElementManifestQueryParams +{ + /// + public required string Name { get; set; } + /// + public required string Value { get; set; } +} + + +/// +/// Source generated JSON parsers for +/// +[JsonSourceGenerationOptions(JsonSerializerDefaults.Web, Converters = [typeof(FSHAHashConverter)])] +[JsonSerializable(typeof(ManifestInfo))] +public partial class EpicManifestParserJsonContext : JsonSerializerContext; + + +/// +/// Extension methods for manifest related things +/// +public static partial class ManifestExtensions +{ + [GeneratedRegex(@"(\d+(?:\.\d+)+)-CL-(\d+)", RegexOptions.Singleline | RegexOptions.IgnoreCase)] + internal static partial Regex VersionAndClRegex(); + + /// + /// Attempts to parse and from the . + /// + /// + /// + /// + /// if was successfully parsed; otherwise, . + public static bool TryParseVersionAndCL(string buildVersion, [NotNullWhen(true)] out Version? version, out int cl) + { + version = null; + cl = -1; + if (string.IsNullOrEmpty(buildVersion)) + return false; + + var match = VersionAndClRegex().Match(buildVersion); + if (!match.Success) + return false; + + version = Version.Parse(match.Groups[1].ValueSpan); + cl = int.Parse(match.Groups[2].ValueSpan); + return true; + } + + /// + /// Reads the HTTP content and returns the value that results from deserializing the content as JSON in an asynchronous operation. + /// + /// The content to read from. + /// A cancellation token that can be used by other objects or threads to receive notice of cancellation. + /// The task object representing the asynchronous operation. + public static Task ReadManifestInfoAsync(this HttpContent content, CancellationToken cancellationToken = default) + => content.ReadFromJsonAsync(EpicManifestParserJsonContext.Default.ManifestInfo, cancellationToken); +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/Decompressor/ManifestZlibDotNetDecompressor.cs b/DeveLanCacheUI_Backend.EpicManifestParser/Decompressor/ManifestZlibDotNetDecompressor.cs new file mode 100644 index 0000000..33ddbab --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/Decompressor/ManifestZlibDotNetDecompressor.cs @@ -0,0 +1,73 @@ +using System.IO.Compression; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.Decompressor; + +/// +/// A decompressor using . +/// +public static class ManifestZlibDotNetDecompressor +{ + ///// + ///// Decompresses data buffer into destination buffer. + ///// + ///// if the decompression was successful; otherwise, . + //public static bool Decompress(object? state, byte[] source, int sourceOffset, int sourceLength, byte[] destination, int destinationOffset, int destinationLength) + //{ + // var zlibng = (Zlibng)state!; + + // var result = zlibng.Uncompress(destination.AsSpan(destinationOffset, destinationLength), + // source.AsSpan(sourceOffset, sourceLength), out int bytesWritten); + + // return result == ZlibngCompressionResult.Ok && bytesWritten == destinationLength; + //} + + /// + /// Decompresses into . + /// + /// + /// • The parameter is kept only to preserve the + /// original delegate/signature; it is not used here.
+ /// • Works on .NET 6+ with ZLibStream. + /// For earlier versions the code falls back to DeflateStream. + /// + /// + /// true when exactly bytes were + /// written; otherwise false. + /// + public static bool Decompress( + object? state, + byte[] source, + int sourceOffset, + int sourceLength, + byte[] destination, + int destinationOffset, + int destinationLength) + { + // Wrap the compressed segment in a read-only MemoryStream. + using var input = new MemoryStream(source, sourceOffset, sourceLength, writable: false); + +#if NET6_0_OR_GREATER + // ZLibStream understands the standard zlib header/footer. + using var decompressor = new ZLibStream(input, CompressionMode.Decompress, leaveOpen: false); +#else + // Older runtimes: fall back to raw deflate (no zlib header). + using var decompressor = new DeflateStream(input, CompressionMode.Decompress, leaveOpen: false); +#endif + + int totalRead = 0; + while (totalRead < destinationLength) + { + int read = decompressor.Read( + destination, + destinationOffset + totalRead, + destinationLength - totalRead); + + if (read == 0) + break; // Stream ended prematurely. + + totalRead += read; + } + + return totalRead == destinationLength; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/DeveLanCacheUI_Backend.EpicManifestParser.csproj b/DeveLanCacheUI_Backend.EpicManifestParser/DeveLanCacheUI_Backend.EpicManifestParser.csproj new file mode 100644 index 0000000..2d11934 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/DeveLanCacheUI_Backend.EpicManifestParser.csproj @@ -0,0 +1,16 @@ + + + + net9.0 + enable + enable + + + + + + + + + + diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs b/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs new file mode 100644 index 0000000..f621fa6 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs @@ -0,0 +1,7 @@ +//namespace DeveLanCacheUI_Backend.EpicManifestParser +//{ +// public class EpicManifestParser +// { +// public async Task<> +// } +//} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/GlobalUsings.cs b/DeveLanCacheUI_Backend.EpicManifestParser/GlobalUsings.cs new file mode 100644 index 0000000..6d9fbc5 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/GlobalUsings.cs @@ -0,0 +1,11 @@ +#if NET9_0_OR_GREATER +global using LockObject = System.Threading.Lock; +global using ManifestData = System.Span; +global using ManifestReader = GenericReader.GenericSpanReader; +global using ManifestRoData = System.ReadOnlySpan; +#else +global using ManifestData = System.Memory; +global using ManifestRoData = System.ReadOnlyMemory; +global using ManifestReader = GenericReader.GenericBufferReader; +global using LockObject = System.Object; +#endif diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/Json/BlobString.cs b/DeveLanCacheUI_Backend.EpicManifestParser/Json/BlobString.cs new file mode 100644 index 0000000..9638196 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/Json/BlobString.cs @@ -0,0 +1,59 @@ +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.Json; + +[DebuggerDisplay("Value,nq")] +internal readonly struct BlobString where T : struct +{ + public T Value { get; } + public BlobString(T value) => Value = value; + + public static BlobString? Parse(ReadOnlySpan source) + { + if (source.Length == 0) return null; + T result = default; + var dest = MemoryMarshal.CreateSpan(ref Unsafe.As(ref result), Unsafe.SizeOf()); + + // Make sure the buffer is at least half the size and that the string is an + // even number of characters long + if (dest.Length >= (uint32)(source.Length / 3) && source.Length % 3 == 0) + { + Span convBuffer = stackalloc uint8[4]; + convBuffer[3] = 0; + + int32 WriteIndex = 0; + // Walk the string 3 chars at a time + for (int32 Index = 0; Index < source.Length; Index += 3, WriteIndex++) + { + convBuffer[0] = source[Index]; + convBuffer[1] = source[Index + 1]; + convBuffer[2] = source[Index + 2]; + dest[WriteIndex] = uint8.Parse(convBuffer); + } + return result; + } + return null; + } + + public static implicit operator BlobString(T value) + { + return new BlobString(value); + } + + public static explicit operator T?(BlobString? holder) + { + return holder?.Value; + } +} + +internal sealed class BlobStringConverter : JsonConverter?> where T : struct +{ + public override BlobString? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + => BlobString.Parse(reader.ValueSpan); + public override void Write(Utf8JsonWriter writer, BlobString? value, JsonSerializerOptions options) + => throw new NotSupportedException(); +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/Json/JsonNodeExtensions.cs b/DeveLanCacheUI_Backend.EpicManifestParser/Json/JsonNodeExtensions.cs new file mode 100644 index 0000000..7d9dd91 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/Json/JsonNodeExtensions.cs @@ -0,0 +1,54 @@ +using DeveLanCacheUI_Backend.EpicManifestParser.UE; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.Json; + +internal static class JsonNodeExtensions +{ + private static readonly JsonSerializerOptions SerializerOptions = new() + { + Converters = + { + new FGuidConverter(), + new FSHAHashConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + new BlobStringConverter(), + } + }; + + public static T GetBlob(this JsonNode? node, T defaultValue = default) where T : struct + { + return node.Deserialize?>(SerializerOptions)?.Value ?? defaultValue; + } + + public static FGuid GetFGuid(this JsonNode? node) + { + return node.Deserialize(SerializerOptions); + } + + public static FSHAHash GetSha(this JsonNode? node) + { + return node.Deserialize(SerializerOptions); + } + + public static string GetString(this JsonNode? node, string defaultValue = "") + { + return node?.GetValue() ?? defaultValue; + } + + public static T Get(this JsonNode? node, T defaultValue = default!) + { + return node is null ? defaultValue : node.GetValue(); + } + + public static T Parse(this JsonNode? node, T defaultValue = default!) + { + return node is null ? defaultValue : node.Deserialize() ?? defaultValue; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/ManifestParseOptions.cs b/DeveLanCacheUI_Backend.EpicManifestParser/ManifestParseOptions.cs new file mode 100644 index 0000000..b491754 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/ManifestParseOptions.cs @@ -0,0 +1,87 @@ +using System.Net; + +namespace DeveLanCacheUI_Backend.EpicManifestParser; +// ReSharper disable UseSymbolAlias + +/// +/// Options/Configuration for parsing manifests +/// +public class ManifestParseOptions +{ + /// + /// Zlib decompress delegate. + /// + public delegate bool DecompressDelegate(object? state, byte[] source, int sourceOffset, int sourceLength, byte[] destination, int destinationOffset, int destinationLength); + + /// + /// Zlib decompress delegate, defaults to . + /// + public DecompressDelegate? Decompressor { get; set; } = ManifestZlibStreamDecompressor.Decompress; + + /// + /// Optional state that gets passed to the delegate. + /// + public object? DecompressorState { get; set; } + + /// + /// Required for downloading, must have a leading slash! + /// + /// + /// Example: http://epicgames-download1.akamaized.net/Builds/Fortnite/CloudDir/
+ /// Distributionpoints can be found here: here. + /// + public string? ChunkBaseUrl { get; set; } + + /// + /// Your own (optional) used for downloading, must not have a ! + /// + public HttpClient? Client { get; set; } + + /// + /// Buffer size for downloading chunks, defaults to 2097152 bytes (2 MiB). + /// + public int ChunkDownloadBufferSize { get; set; } = 2097152; + + /// + /// Optional for caching chunks, very recommended. + /// + public string? ChunkCacheDirectory { get; set; } + + /// + /// Whether or not to cache the chunks 1:1 as they were downloaded, defaults to . + /// + public bool CacheChunksAsIs { get; set; } + + /// + /// Optional for caching manifests when using . + /// + public string? ManifestCacheDirectory { get; set; } + + /// + /// Creates a default and also sets to its instance. + /// + /// The created . + public HttpClient CreateDefaultClient() + { + if (Client is not null) + return Client; + + var handler = new SocketsHttpHandler + { + UseCookies = false, + UseProxy = false, + AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip, + MaxConnectionsPerServer = 256 + }; + Client = new HttpClient(handler) + { + DefaultRequestVersion = new Version(1, 1), + DefaultVersionPolicy = HttpVersionPolicy.RequestVersionExact, + Timeout = TimeSpan.FromSeconds(30) + }; + Client.DefaultRequestHeaders.Accept.ParseAdd("*/*"); + Client.DefaultRequestHeaders.UserAgent.ParseAdd("EpicGamesLauncher/16.13.0-36938137+++Portal+Release-Live Windows/10.0.26100.1.256.64bit"); + Client.DefaultRequestHeaders.ConnectionClose = false; + return Client; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/ManifestZlibStreamDecompressor.cs b/DeveLanCacheUI_Backend.EpicManifestParser/ManifestZlibStreamDecompressor.cs new file mode 100644 index 0000000..710f6c1 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/ManifestZlibStreamDecompressor.cs @@ -0,0 +1,22 @@ +using System.IO.Compression; + +namespace DeveLanCacheUI_Backend.EpicManifestParser; + +/// +/// The default decompressor using . +/// +public static class ManifestZlibStreamDecompressor +{ + /// + /// Decompresses data buffer into destination buffer. + /// + /// if the decompression was successful; otherwise, . + public static bool Decompress(object? state, byte[] source, int sourceOffset, int sourceLength, byte[] destination, int destinationOffset, int destinationLength) + { + using var destinationMs = new MemoryStream(destination, destinationOffset, destinationLength, true, true); + using var sourceMs = new MemoryStream(source, sourceOffset, sourceLength, false, true); + using var zlibStream = new ZLibStream(sourceMs, CompressionMode.Decompress); + zlibStream.CopyTo(destinationMs); + return destinationMs.Position == destinationLength; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkDataListVersion.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkDataListVersion.cs new file mode 100644 index 0000000..ea3e9f5 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkDataListVersion.cs @@ -0,0 +1,10 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal enum EChunkDataListVersion : uint8 +{ + Original = 0, + + // Always after the latest version, signifies the latest version plus 1 to allow initialization simplicity. + LatestPlusOne, + Latest = LatestPlusOne - 1 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkHashFlags.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkHashFlags.cs new file mode 100644 index 0000000..efa5522 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkHashFlags.cs @@ -0,0 +1,13 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +[Flags] +internal enum EChunkHashFlags : uint8 +{ + None = 0x00, + + // Flag for FRollingHash class used, stored in RollingHash on header. + RollingPoly64 = 0x01, + + // Flag for FSHA1 class used, stored in SHAHash on header. + Sha1 = 0x02, +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkStorageFlags.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkStorageFlags.cs new file mode 100644 index 0000000..7e610da --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkStorageFlags.cs @@ -0,0 +1,13 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +[Flags] +internal enum EChunkStorageFlags : uint8 +{ + None = 0x00, + + // Flag for compressed data. + Compressed = 0x01, + + // Flag for encrypted. If also compressed, decrypt first. Encryption will ruin compressibility. + Encrypted = 0x02, +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkVersion.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkVersion.cs new file mode 100644 index 0000000..10dba05 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EChunkVersion.cs @@ -0,0 +1,13 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal enum EChunkVersion : uint32 +{ + Invalid = 0, + Original, + StoresShaAndHashType, + StoresDataSizeUncompressed, + + // Always after the latest version, signifies the latest version plus 1 to allow initialization simplicity. + LatestPlusOne, + Latest = LatestPlusOne - 1 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFeatureLevel.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFeatureLevel.cs new file mode 100644 index 0000000..6c44335 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFeatureLevel.cs @@ -0,0 +1,133 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE EFeatureLevel enum +/// +public enum EFeatureLevel +{ + /// + /// The original version. + /// + Original = 0, + /// + /// Support for custom fields. + /// + CustomFields, + /// + /// Started storing the version number. + /// + StartStoringVersion, + /// + /// Made after data files where renamed to include the hash value, these chunks now go to ChunksV2. + /// + DataFileRenames, + /// + /// Manifest stores whether build was constructed with chunk or file data. + /// + StoresIfChunkOrFileData, + /// + /// Manifest stores group number for each chunk/file data for reference so that external readers don't need to know how to calculate them. + /// + StoresDataGroupNumbers, + /// + /// Added support for chunk compression, these chunks now go to ChunksV3. NB: Not File Data Compression yet. + /// + ChunkCompressionSupport, + /// + /// Manifest stores product prerequisites info. + /// + StoresPrerequisitesInfo, + /// + /// Manifest stores chunk download sizes. + /// + StoresChunkFileSizes, + /// + /// Manifest can optionally be stored using UObject serialization and compressed. + /// + StoredAsCompressedUClass, + /// + /// Removed and never used. + /// + UNUSED_0, + /// + /// Removed and never used. + /// + UNUSED_1, + /// + /// Manifest stores chunk data SHA1 hash to use in place of data compare, for faster generation. + /// + StoresChunkDataShaHashes, + /// + /// Manifest stores Prerequisite Ids. + /// + StoresPrerequisiteIds, + /// + /// The first minimal binary format was added. UObject classes will no longer be saved out when binary selected. + /// + StoredAsBinaryData, + /// + /// Temporary level where manifest can reference chunks with dynamic window size, but did not serialize them. Chunks from here onwards are stored in ChunksV4. + /// + VariableSizeChunksWithoutWindowSizeChunkInfo, + /// + /// Manifest can reference chunks with dynamic window size, and also serializes them. + /// + VariableSizeChunks, + /// + /// Manifest uses a build id generated from its metadata. + /// + UsesRuntimeGeneratedBuildId, + /// + /// Manifest uses a build id generated unique at build time, and stored in manifest. + /// + UsesBuildTimeGeneratedBuildId, + + /// + /// Undocumented in UE + /// + Unknown1, + /// + /// Undocumented in UE + /// + Unknown2, + /// + /// Used for fortnite currently + /// + Unknown3, + + /// + /// !! Always after the latest version entry, signifies the latest version plus 1 to allow the following Latest alias. + /// + LatestPlusOne = UsesBuildTimeGeneratedBuildId + 1, + /// + /// An alias for the actual latest version value. + /// + Latest = LatestPlusOne - 1, + /// + /// An alias to provide the latest version of a manifest supported by file data (nochunks). + /// + LatestNoChunks = StoresChunkFileSizes, + /// + /// An alias to provide the latest version of a manifest supported by a json serialized format. + /// + LatestJson = StoresPrerequisiteIds, + /// + /// An alias to provide the first available version of optimised delta manifest saving. + /// + FirstOptimisedDelta = UsesRuntimeGeneratedBuildId, + + /// + /// More aliases, but this time for values that have been renamed + /// + StoresUniqueBuildId = UsesRuntimeGeneratedBuildId, + + /// + /// JSON manifests were stored with a version of 255 during a certain CL range due to a bug. + /// We will treat this as being StoresChunkFileSizes in code. + /// + BrokenJsonVersion = 255, + /// + /// This is for UObject default, so that we always serialize it. + /// + Invalid = -1 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileManifestListVersion.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileManifestListVersion.cs new file mode 100644 index 0000000..71062e1 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileManifestListVersion.cs @@ -0,0 +1,10 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal enum EFileManifestListVersion : uint8 +{ + Original = 0, + + // Always after the latest version, signifies the latest version plus 1 to allow initialization simplicity. + LatestPlusOne, + Latest = LatestPlusOne - 1 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileMetaFlags.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileMetaFlags.cs new file mode 100644 index 0000000..d0971ae --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EFileMetaFlags.cs @@ -0,0 +1,25 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE EFileMetaFlags enum +/// +[Flags] +public enum EFileMetaFlags : uint8 +{ + /// + /// None + /// + None = 0, + /// + /// Flag for readonly file. + /// + ReadOnly = 1, + /// + /// Flag for natively compressed. + /// + Compressed = 1 << 1, + /// + /// Flag for unix executable. + /// + UnixExecutable = 1 << 2 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestMetaVersion.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestMetaVersion.cs new file mode 100644 index 0000000..2cacf4c --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestMetaVersion.cs @@ -0,0 +1,11 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal enum EManifestMetaVersion : uint8 +{ + Original = 0, + SerialisesBuildId, + + // Always after the latest version, signifies the latest version plus 1 to allow initialization simplicity. + LatestPlusOne, + Latest = LatestPlusOne - 1 +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestStorageFlags.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestStorageFlags.cs new file mode 100644 index 0000000..de35dd6 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/EManifestStorageFlags.cs @@ -0,0 +1,12 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +[Flags] +internal enum EManifestStorageFlags : uint8 +{ + // Stored as raw data. + None = 0, + // Flag for compressed data. + Compressed = 1, + // Flag for encrypted. If also compressed, decrypt first. Encryption will ruin compressibility. + Encrypted = 1 << 1, +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FBuildPatchAppManifest.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FBuildPatchAppManifest.cs new file mode 100644 index 0000000..975e9ae --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FBuildPatchAppManifest.cs @@ -0,0 +1,508 @@ +using AsyncKeyedLock; +using DeveLanCacheUI_Backend.EpicManifestParser.Json; +using System.Buffers; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.InteropServices; +using System.Text.Json; +using System.Text.Json.Nodes; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FBuildPatchAppManifest struct +/// +public class FBuildPatchAppManifest +{ + /// + public FManifestMeta Meta { get; internal set; } = null!; + /// + public IReadOnlyList ChunkList { get; internal set; } = null!; + /// + public IReadOnlyList Files { get; internal set; } = null!; + /// + public IReadOnlyList CustomFields { get; internal set; } = null!; + /// + public IReadOnlyDictionary Chunks { get; internal set; } = null!; + + /// + public int64 TotalBuildSize { get; internal set; } + /// + public int64 TotalDownloadSize { get; internal set; } + + internal ManifestParseOptions Options { get; init; } = null!; + internal AsyncKeyedLocker ChunksLocker { get; set; } = null!; + + internal FBuildPatchAppManifest() { } + + /// + /// Finds a file by . + /// + /// The filename to find. + /// The type to compare the filename. + /// The instance if the the file was found; otherwise, . + public FFileManifest? FindFile(string fileName, StringComparison comparisonType = StringComparison.Ordinal) + => TryFindFile(fileName, comparisonType, out var file) ? file : null; + + /// + /// Tries to find a file by using to compare it. + /// + /// The filename to find. + /// The find result. + /// if the the file was found; otherwise, . + public bool TryFindFile(string fileName, [NotNullWhen(true)] out FFileManifest? fileManifest) + => TryFindFile(fileName, StringComparison.Ordinal, out fileManifest); + + /// + /// Tries to find a file by . + /// + /// The filename to find. + /// The type to compare the filename. + /// The find result. + /// if the the file was found; otherwise, . + public bool TryFindFile(string fileName, StringComparison comparisonType, [NotNullWhen(true)] out FFileManifest? fileManifest) + { + foreach (var file in Files) + { + if (!file.FileName.Equals(fileName, comparisonType)) + continue; + + fileManifest = file; + return true; + } + + fileManifest = null; + return false; + } + + /// + /// Get the chunk sub-directory name + /// + public string GetChunkSubdir() => Meta.FeatureLevel switch + { + > EFeatureLevel.StoredAsBinaryData => "ChunksV4", + > EFeatureLevel.StoresDataGroupNumbers => "ChunksV3", + > EFeatureLevel.StartStoringVersion => "ChunksV2", + _ => "Chunks" + }; + + /// + /// Helper function to decide whether the passed in data is a JSON string we expect to deserialize a manifest from + /// + /// if the is JSON; otherwise, . + public static bool IsJson(ManifestRoData dataInput) + { + // The best we can do is look for the mandatory first character open curly brace, + // it will be within the first 4 characters (may have BOM) + var span = dataInput +#if !NET9_0_OR_GREATER + .Span +#endif + ; + for (var idx = 0; idx < 4 && idx < span.Length; ++idx) + { + if (span[idx] == '{') + { + return true; + } + } + return false; + } + + /// + /// Deserializes a binary or JSON manifest + /// + /// + public static FBuildPatchAppManifest Deserialize(ManifestRoData dataInput, Action? optionsBuilder = null) + { + var options = new ManifestParseOptions(); + optionsBuilder?.Invoke(options); + return Deserialize(dataInput, options); + } + + /// + /// Deserializes a JSON manifest + /// + /// The span to parse from + /// Builder for options/configuration to parse + public static FBuildPatchAppManifest DeserializeJson(ManifestRoData dataInput, Action? optionsBuilder = null) + { + var options = new ManifestParseOptions(); + optionsBuilder?.Invoke(options); + return DeserializeJson(dataInput, options); + } + + /// + /// Deserializes a binary manifest + /// + /// The span to parse from + /// Builder for options/configuration to parse + /// Manifest is encrypted or older than + /// Data is compressed and zlib-ng instance was null + /// Error while parsing + /// Hashes do not match + public static FBuildPatchAppManifest DeserializeBinary(ManifestRoData dataInput, Action? optionsBuilder = null) + { + var options = new ManifestParseOptions(); + optionsBuilder?.Invoke(options); + return DeserializeBinary(dataInput, options); + } + + /// + /// Deserializes a binary or JSON manifest + /// + /// + public static FBuildPatchAppManifest Deserialize(ManifestRoData dataInput, ManifestParseOptions options) + { + return IsJson(dataInput) ? DeserializeJson(dataInput, options) : DeserializeBinary(dataInput, options); + } + + /// + /// Deserializes a JSON manifest + /// + /// The span to parse from + /// Options/Configuration to parse + public static FBuildPatchAppManifest DeserializeJson(ManifestRoData dataInput, ManifestParseOptions options) + { + var reader = JsonNode.Parse(dataInput +#if !NET9_0_OR_GREATER + .Span +#endif + )!.AsObject(); + + var featureLevel = reader["ManifestFileVersion"].GetBlob(EFeatureLevel.CustomFields); + if (featureLevel == EFeatureLevel.BrokenJsonVersion) + featureLevel = EFeatureLevel.StoresChunkFileSizes; + + var meta = new FManifestMeta + { + FeatureLevel = featureLevel, + AppID = reader["AppID"].GetBlob(), + AppName = reader["AppNameString"].GetString(), + BuildVersion = reader["BuildVersionString"].GetString(), + LaunchExe = reader["LaunchExeString"].GetString(), + LaunchCommand = reader["LaunchCommand"].GetString(), + PrereqName = reader["PrereqName"].GetString(), + PrereqPath = reader["PrereqPath"].GetString(), + PrereqArgs = reader["PrereqArgs"].GetString(), + UninstallExe = "", + UninstallCommand = "", + }; + + var jsonFileManifestList = reader["FileManifestList"]!.AsArray(); + var fileManifests = new FFileManifest[jsonFileManifestList.Count]; + var fileManifestsSpan = fileManifests.AsSpan(); + + //var allDataGuids = new HashSet(); + var mutableChunkInfoLookup = new Dictionary(); + + for (var i = 0; i < fileManifestsSpan.Length; i++) + { + var jsonFileManifest = jsonFileManifestList[i]!; + var fileManifest = fileManifestsSpan[i] = new FFileManifest + { + FileName = jsonFileManifest["Filename"].GetString(), + FileHash = jsonFileManifest["FileHash"].GetBlob(), + InstallTags = jsonFileManifest["InstallTags"].Parse([]), + SymlinkTarget = jsonFileManifest["SymlinkTarget"].GetString() + }; + var jsonFileChunkParts = jsonFileManifest["FileChunkParts"]!.AsArray(); + fileManifest.ChunkPartsArray = new FChunkPart[jsonFileChunkParts.Count]; + var chunkPartsSpan = fileManifest.ChunkPartsArray.AsSpan(); + for (var j = 0; j < chunkPartsSpan.Length; j++) + { + var jsonFileChunkPart = jsonFileChunkParts[j]!; + var chunkPartGuid = jsonFileChunkPart["Guid"].GetFGuid(); + var chunkPartOffset = jsonFileChunkPart["Offset"].GetBlob(); + var chunkPartSize = jsonFileChunkPart["Size"].GetBlob(); + chunkPartsSpan[j] = new FChunkPart(chunkPartGuid, chunkPartOffset, chunkPartSize); + + ref var lookupChunk = ref CollectionsMarshal.GetValueRefOrAddDefault(mutableChunkInfoLookup, chunkPartGuid, out var exists); + if (!exists) + { + lookupChunk = new FChunkInfo + { + Guid = chunkPartGuid + }; + } + } + + if (jsonFileManifest["bIsUnixExecutable"].Get()) + fileManifest.FileMetaFlags |= EFileMetaFlags.UnixExecutable; + if (jsonFileManifest["bIsReadOnly"].Get()) + fileManifest.FileMetaFlags |= EFileMetaFlags.ReadOnly; + if (jsonFileManifest["bIsCompressed"].Get()) + fileManifest.FileMetaFlags |= EFileMetaFlags.Compressed; + } + + var chunkList = new FChunkInfo[mutableChunkInfoLookup.Count]; + var chunkListSpan = chunkList.AsSpan(); + var chunkIndex = 0; + foreach (var chunk in mutableChunkInfoLookup.Values) + { + chunkListSpan[chunkIndex++] = chunk; + } + + var hasChunkHashList = false; + var jsonChunkHashListNode = reader["ChunkHashList"]; + if (jsonChunkHashListNode is not null) + { + var jsonChunkHashList = jsonChunkHashListNode.AsObject(); + + foreach (var (guidString, jsonChunkHash) in jsonChunkHashList) + { + var guid = new FGuid(guidString); + var chunkHash = jsonChunkHash.GetBlob(); + mutableChunkInfoLookup[guid].Hash = chunkHash; + } + + hasChunkHashList = true; + } + + var jsonChunkShaListNode = reader["ChunkShaList"]; + if (jsonChunkShaListNode is not null) + { + var jsonChunkShaList = jsonChunkShaListNode.AsObject(); + + foreach (var (guidString, jsonSha) in jsonChunkShaList) + { + var guid = new FGuid(guidString); + var chunkSha = jsonSha.GetSha(); + mutableChunkInfoLookup[guid].ShaHash = chunkSha; + } + } + + var prereqIds = reader["PrereqIds"].Deserialize(); + if (prereqIds is null) + { + // TODO: https://github.com/EpicGames/UnrealEngine/blob/8c31706601135aadf2f957fb76e2af46f04a8ef9/Engine/Source/Runtime/Online/BuildPatchServices/Private/BuildPatchManifest.cpp#L602 + meta.PrereqIds = []; + } + else + { + meta.PrereqIds = prereqIds; + } + + var jsonDataGroupListNode = reader["DataGroupList"]; + if (jsonDataGroupListNode is not null) + { + var jsonDataGroupList = jsonDataGroupListNode.AsObject(); + + foreach (var (guidString, jsonDataGroup) in jsonDataGroupList) + { + var guid = new FGuid(guidString); + var dataGroup = jsonDataGroup.GetBlob(); + mutableChunkInfoLookup[guid].GroupNumber = dataGroup; + } + } + else + { + // TODO: https://github.com/EpicGames/UnrealEngine/blob/8c31706601135aadf2f957fb76e2af46f04a8ef9/Engine/Source/Runtime/Online/BuildPatchServices/Private/BuildPatchManifest.cpp#L635 + // https://github.com/EpicGames/UnrealEngine/blob/8c31706601135aadf2f957fb76e2af46f04a8ef9/Engine/Source/Runtime/Core/Private/Misc/Crc.cpp#L592 + } + + var hasChunkFilesizeList = false; + var jsonChunkFilesizeListNode = reader["ChunkFilesizeList"]; + if (jsonChunkFilesizeListNode is not null) + { + var jsonChunkFilesizeList = jsonChunkFilesizeListNode.AsObject(); + + foreach (var (guidString, jsonFileSize) in jsonChunkFilesizeList) + { + var guid = new FGuid(guidString); + var fileSize = jsonFileSize.GetBlob(); + mutableChunkInfoLookup[guid].FileSize = fileSize; + } + + hasChunkFilesizeList = true; + } + + if (!hasChunkFilesizeList) + { + // Missing chunk list, version before we saved them compressed. Assume original fixed chunk size of 1 MiB. + foreach (var chunk in chunkListSpan) + { + chunk.FileSize = 1048576; + } + } + + if (reader.TryGetPropertyValue("bIsFileData", out var jsonIsFileData)) + { + meta.bIsFileData = jsonIsFileData.Get(); + } + else + { + meta.bIsFileData = !hasChunkHashList; + } + + FCustomField[]? customFields = null; + var jsonCustomFieldsNode = reader["CustomFields"]; + if (jsonCustomFieldsNode is not null) + { + var jsonCustomFields = jsonCustomFieldsNode.AsObject(); + customFields = new FCustomField[jsonCustomFields.Count]; + var customFieldIndex = 0; + + foreach (var (name, jsonValue) in jsonCustomFields) + { + customFields[customFieldIndex++] = new FCustomField + { + Name = name, + Value = jsonValue.GetString() + }; + } + } + + meta.BuildId = FManifestMeta.GetBackwardsCompatibleBuildId(meta); + + var manifest = new FBuildPatchAppManifest + { + Meta = meta, + ChunkList = chunkList, + Files = fileManifests, + CustomFields = customFields ?? [], + Chunks = mutableChunkInfoLookup, + Options = options + }; + manifest.PostSetup(); + + // FileDataList.OnPostLoad(); + { + Array.Sort(fileManifests); + for (var i = 0; i < fileManifestsSpan.Length; i++) + { + var file = fileManifestsSpan[i]; + file.Manifest = manifest; + foreach (var chunkPart in file.ChunkPartsArray.AsSpan()) + { + file.FileSize += chunkPart.Size; + } + } + } + + return manifest; + } + + /// + /// Deserializes a binary manifest + /// + /// The span to parse from + /// Options/Configuration to parse + /// Manifest is encrypted or older than + /// Data is compressed and zlib-ng instance was null + /// Error while parsing + /// Hashes do not match + public static FBuildPatchAppManifest DeserializeBinary(ManifestRoData dataInput, ManifestParseOptions options) + { + var fileReader = new ManifestReader(dataInput); + byte[]? manifestRawDataBuffer = null; + + try + { + var header = new FManifestHeader(ref fileReader); + + if (header.Version < EFeatureLevel.StoredAsBinaryData) + throw new NotSupportedException("Manifests below feature level StoredAsBinaryData are not supported"); + if (header.StoredAs.HasFlag(EManifestStorageFlags.Encrypted)) + throw new NotSupportedException("Encrypted manifests are not supported"); + if (header.StoredAs.HasFlag(EManifestStorageFlags.Compressed) && options.Decompressor is null) + throw new InvalidOperationException("Data is compressed and decompressor delegate was null"); + + ManifestData manifestRawData; + + if (header.StoredAs.HasFlag(EManifestStorageFlags.Compressed)) + { + manifestRawDataBuffer = ArrayPool.Shared.Rent(header.DataSizeCompressed + header.DataSizeUncompressed); + manifestRawData = manifestRawDataBuffer +#if NET9_0_OR_GREATER + .AsSpan +#else + .AsMemory +#endif + (header.DataSizeCompressed, header.DataSizeUncompressed); + + var manifestCompressedData = manifestRawDataBuffer.AsSpan(0, header.DataSizeCompressed); + fileReader.Read(manifestCompressedData); + + var result = options.Decompressor!.Invoke( + options.DecompressorState, + manifestRawDataBuffer, 0, header.DataSizeCompressed, + manifestRawDataBuffer, header.DataSizeCompressed, header.DataSizeUncompressed); + if (!result) + throw new FileLoadException("Failed to uncompress data"); + } + else if (header.StoredAs == EManifestStorageFlags.None) + { + manifestRawDataBuffer = ArrayPool.Shared.Rent(header.DataSizeCompressed); + manifestRawData = manifestRawDataBuffer +#if NET9_0_OR_GREATER + .AsSpan +#else + .AsMemory +#endif + (0, header.DataSizeCompressed); + fileReader.Read(manifestRawData +#if !NET9_0_OR_GREATER + .Span +#endif + ); + } + else + { + throw new UnreachableException("Manifest has invalid or unknown storage flags"); + } + + var hash = FSHAHash.Compute(manifestRawData +#if !NET9_0_OR_GREATER + .Span +#endif + ); + if (header.SHAHash != hash) + throw new InvalidDataException($"Hash does not match. expected: {header.SHAHash}, actual: {hash}"); + + var reader = new ManifestReader(manifestRawData); + var chunks = new Dictionary(); + var manifest = new FBuildPatchAppManifest + { + Chunks = chunks, + Options = options + }; + manifest.Meta = new FManifestMeta(ref reader); + manifest.ChunkList = FChunkInfo.ReadChunkDataList(ref reader, chunks); + manifest.Files = FFileManifest.ReadFileDataList(ref reader, manifest); + manifest.CustomFields = FCustomField.ReadCustomFields(ref reader); + manifest.PostSetup(); + return manifest; + } + finally + { + if (manifestRawDataBuffer is not null) + ArrayPool.Shared.Return(manifestRawDataBuffer); + } + } + + private void PostSetup() + { + foreach (var file in Files) + { + TotalBuildSize += file.FileSize; + } + + foreach (var chunk in ChunkList) + { + TotalDownloadSize += chunk.FileSize; + } + + if (!string.IsNullOrEmpty(Options.ChunkBaseUrl)) + { + ChunksLocker = new AsyncKeyedLocker(lockerOptions => + { + lockerOptions.MaxCount = 1; + lockerOptions.PoolSize = 128; + lockerOptions.PoolInitialFill = 64; + }); + + Options.CreateDefaultClient(); + } + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkHeader.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkHeader.cs new file mode 100644 index 0000000..4e80656 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkHeader.cs @@ -0,0 +1,113 @@ +using System.Runtime.CompilerServices; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal struct FChunkHeader +{ + public const uint32 Magic = 0xB1FE3AA2; + + /// + /// The version of this header data. + /// + public EChunkVersion Version; + /// + /// The size of this header. + /// + public int32 HeaderSize; + /*/// + /// The GUID for this data. + /// + public FGuid Guid;*/ + /// + /// The size of this data compressed. + /// + public int32 DataSizeCompressed; + /// + /// The size of this data uncompressed. + /// + public int32 DataSizeUncompressed; + /// + /// How the chunk data is stored. + /// + public EChunkStorageFlags StoredAs; + /*/// + /// What type of hash we are using. + /// + public EChunkHashFlags HashType; + /// + /// The FRollingHash hashed value for this chunk data. + /// + public uint64 RollingHash; + /// + /// The FSHA hashed value for this chunk data. + /// + public FSHAHash SHAHash;*/ + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static FChunkHeader Parse(ManifestData data) + { + var reader = new ManifestReader(data); + return new FChunkHeader(ref reader); + } + + internal FChunkHeader(ref ManifestReader reader) + { + var startPos = reader.Position; + var archiveSizeLeft = reader.Length - startPos; + var versionSizesSpan = ChunkHeaderVersionSizes.AsSpan(); + var expectedSerializedBytes = versionSizesSpan[(int32)EChunkVersion.Original]; + + if (archiveSizeLeft >= expectedSerializedBytes) + { + var magic = reader.Read(); + if (magic != Magic) + throw new FileLoadException($"invalid chunk magic: 0x{magic:X}"); + + Version = reader.Read(); + HeaderSize = reader.Read(); + DataSizeCompressed = reader.Read(); + + //Guid = reader.Read(); + //RollingHash = reader.Read(); + reader.Position += FGuid.Size + sizeof(uint64); + + StoredAs = reader.Read(); + DataSizeUncompressed = 1024 * 1024; + + if (Version >= EChunkVersion.StoresShaAndHashType) + { + expectedSerializedBytes = versionSizesSpan[(int32)EChunkVersion.StoresShaAndHashType]; + if (archiveSizeLeft >= expectedSerializedBytes) + { + //SHAHash = reader.Read(); + //HashType = reader.Read(); + reader.Position += FSHAHash.Size + sizeof(EChunkHashFlags); + } + + if (Version >= EChunkVersion.StoresDataSizeUncompressed) + { + expectedSerializedBytes = versionSizesSpan[(int32)EChunkVersion.StoresDataSizeUncompressed]; + if (archiveSizeLeft >= expectedSerializedBytes) + { + DataSizeUncompressed = reader.Read(); + } + } + } + } + + var success = reader.Position - startPos == expectedSerializedBytes; + reader.Position = startPos + HeaderSize; + } + + private static readonly uint32[] ChunkHeaderVersionSizes = + [ + // Dummy for indexing. + 0, + // Original is 41 bytes (32b Magic, 32b Version, 32b HeaderSize, 32b DataSizeCompressed, 4x32b GUID, 64b Hash, 8b StoredAs). + 41, + // StoresShaAndHashType is 62 bytes (328b Original, 160b SHA1, 8b HashType). + 62, + // StoresDataSizeUncompressed is 66 bytes (496b StoresShaAndHashType, 32b DataSizeUncompressed). + 66 + ]; +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkInfo.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkInfo.cs new file mode 100644 index 0000000..28e0ff4 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkInfo.cs @@ -0,0 +1,331 @@ +using System.Buffers; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FChunkInfo struct +/// +public sealed class FChunkInfo +{ + /// + /// The GUID for this data. + /// + public FGuid Guid { get; internal set; } + /// + /// The FRollingHash hashed value for this chunk data. + /// + public uint64 Hash { get; internal set; } + /// + /// The FSHA hashed value for this chunk data. + /// + public FSHAHash ShaHash { get; internal set; } + /// + /// The group number this chunk divides into. + /// + public uint8 GroupNumber { get; internal set; } + /// + /// The window size for this chunk. + /// + public uint32 WindowSize { get; internal set; } + /// + /// The file download size for this chunk. + /// + public int64 FileSize { get; internal set; } + + /// + /// + /// Url to download this chunk + /// + public string GetUrl(FBuildPatchAppManifest manifest) => + $"{manifest.Options.ChunkBaseUrl}{manifest.GetChunkSubdir()}/{GroupNumber:D2}/{Hash:X16}_{Guid}.chunk"; + + /// + /// + /// to download this chunk + /// + public Uri GetUri(FBuildPatchAppManifest manifest) => new(GetUrl(manifest), UriKind.Absolute); + + internal string? CachePath { get; set; } + + internal static FChunkInfo[] ReadChunkDataList(ref ManifestReader reader, Dictionary chunksDict) + { + var startPos = reader.Position; + var dataSize = reader.Read(); + var dataVersion = reader.Read(); + var elementCount = reader.Read(); + + var chunks = new FChunkInfo[elementCount]; + var chunksSpan = chunks.AsSpan(); + + chunksDict.EnsureCapacity(elementCount); + + if (dataVersion >= EChunkDataListVersion.Original) + { + for (var i = 0; i < elementCount; i++) + { + var chunk = new FChunkInfo(); + chunk.Guid = reader.Read(); + chunksSpan[i] = chunk; + chunksDict.Add(chunk.Guid, chunk); + } + for (var i = 0; i < elementCount; i++) + chunksSpan[i].Hash = reader.Read(); + for (var i = 0; i < elementCount; i++) + chunksSpan[i].ShaHash = reader.Read(); + for (var i = 0; i < elementCount; i++) + chunksSpan[i].GroupNumber = reader.Read(); + for (var i = 0; i < elementCount; i++) + chunksSpan[i].WindowSize = reader.Read(); + for (var i = 0; i < elementCount; i++) + chunksSpan[i].FileSize = reader.Read(); + } + else + { + var defaultChunk = new FChunkInfo + { + WindowSize = 1048576 + }; + chunksSpan.Fill(defaultChunk); + } + + reader.Position = startPos + dataSize; + return chunks; + } + + [SuppressMessage("ReSharper", "UseSymbolAlias")] + internal async Task ReadDataAsIsAsync(byte[] destination, FBuildPatchAppManifest manifest, CancellationToken cancellationToken = default) + { + var fileSize = 0; + var shouldCache = manifest.Options.ChunkCacheDirectory is not null; + string? cachePath = null; + + if (CachePath is not null) + { + using var fileHandle = File.OpenHandle(CachePath); + fileSize = (int)RandomAccess.GetLength(fileHandle); + await RandomAccess.ReadAsync(fileHandle, destination.AsMemory(0, fileSize), 0, cancellationToken).ConfigureAwait(false); + } + else + { + using var _ = await manifest.ChunksLocker.LockAsync(Guid, cancellationToken).ConfigureAwait(false); + + if (CachePath is not null) + { + using var fileHandle = File.OpenHandle(CachePath); + fileSize = (int)RandomAccess.GetLength(fileHandle); + await RandomAccess.ReadAsync(fileHandle, destination.AsMemory(0, fileSize), 0, cancellationToken).ConfigureAwait(false); + } + else if (shouldCache) + { + cachePath = Path.Combine(manifest.Options.ChunkCacheDirectory!, $"v2_{Hash:X16}_{Guid}.chunk"); + if (File.Exists(cachePath)) + { + CachePath = cachePath; + using var fileHandle = File.OpenHandle(CachePath); + fileSize = (int)RandomAccess.GetLength(fileHandle); + await RandomAccess.ReadAsync(fileHandle, destination.AsMemory(0, fileSize), 0, cancellationToken).ConfigureAwait(false); + } + } + + if (fileSize == 0) + { + var uri = GetUri(manifest); + var destMs = new MemoryStream(destination, 0, destination.Length, true); + using var res = await manifest.Options.Client!.GetAsync(uri, cancellationToken).ConfigureAwait(false); + EnsureSuccessStatusCode(res, uri); + await res.Content.CopyToAsync(destMs, cancellationToken).ConfigureAwait(false); + fileSize = (int)destMs.Position; + + if (shouldCache) + { + using (var fileHandle = File.OpenHandle(cachePath!, FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.None, fileSize)) + { + await RandomAccess.WriteAsync(fileHandle, new ReadOnlyMemory(destination, 0, fileSize), 0, cancellationToken).ConfigureAwait(false); + RandomAccess.FlushToDisk(fileHandle); + } + CachePath = cachePath; + } + } + } + + var header = FChunkHeader.Parse(new ManifestData(destination, 0, fileSize)); + + if (header.StoredAs == EChunkStorageFlags.None) + { + Unsafe.CopyBlockUnaligned(ref destination[0], ref destination[header.HeaderSize], (uint)header.DataSizeCompressed); + return header.DataSizeCompressed; + } + + if (header.StoredAs.HasFlag(EChunkStorageFlags.Encrypted)) + throw new NotSupportedException("Encrypted chunks are not supported"); + if (!header.StoredAs.HasFlag(EChunkStorageFlags.Compressed)) + throw new UnreachableException("Unknown/new chunk ChunkStorageFlag"); + if (manifest.Options.Decompressor is null) + throw new InvalidOperationException("Data is compressed and decompressor delegate was null"); + + // cant uncompress in-place + var poolBuffer = ArrayPool.Shared.Rent(header.DataSizeCompressed); + + try + { + Unsafe.CopyBlockUnaligned(ref poolBuffer[0], ref destination[header.HeaderSize], (uint)header.DataSizeCompressed); + + var result = manifest.Options.Decompressor.Invoke( + manifest.Options.DecompressorState, + poolBuffer, 0, header.DataSizeCompressed, + destination, 0, header.DataSizeUncompressed); + if (!result) + throw new FileLoadException("Failed to uncompress data"); + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + + return header.DataSizeUncompressed; + } + + [SuppressMessage("ReSharper", "UseSymbolAlias")] + internal async Task ReadDataAsync(byte[] buffer, int offset, int count, int chunkPartOffset, FBuildPatchAppManifest manifest, CancellationToken cancellationToken = default) + { + if (CachePath is not null) + { + using var fileHandle = File.OpenHandle(CachePath); + return await RandomAccess.ReadAsync(fileHandle, buffer.AsMemory(offset, count), chunkPartOffset, cancellationToken).ConfigureAwait(false); + } + + using var _ = await manifest.ChunksLocker.LockAsync(Guid, cancellationToken).ConfigureAwait(false); + + if (CachePath is not null) + { + using var fileHandle = File.OpenHandle(CachePath); + return await RandomAccess.ReadAsync(fileHandle, buffer.AsMemory(offset, count), chunkPartOffset, cancellationToken).ConfigureAwait(false); + } + + var shouldCache = manifest.Options.ChunkCacheDirectory is not null; + string? cachePath = null; + + if (shouldCache) + { + cachePath = Path.Combine(manifest.Options.ChunkCacheDirectory!, $"{Hash:X16}_{Guid}.chunk"); + if (File.Exists(cachePath)) + { + CachePath = cachePath; + using var fileHandle = File.OpenHandle(CachePath); + return await RandomAccess.ReadAsync(fileHandle, buffer.AsMemory(offset, count), chunkPartOffset, cancellationToken).ConfigureAwait(false); + } + } + + byte[]? poolBuffer = null; + byte[]? uncompressPoolBuffer = null; + + try + { + var uri = GetUri(manifest); + using var res = await manifest.Options.Client!.GetAsync(uri, cancellationToken).ConfigureAwait(false); + EnsureSuccessStatusCode(res, uri); + var poolBufferSize = res.Content.Headers.ContentLength ?? manifest.Options.ChunkDownloadBufferSize; + poolBuffer = ArrayPool.Shared.Rent((int)poolBufferSize); + var destMs = new MemoryStream(poolBuffer, 0, poolBuffer.Length, true, true); + await res.Content.CopyToAsync(destMs, cancellationToken).ConfigureAwait(false); + var responseSize = (int)destMs.Length; + + var header = FChunkHeader.Parse(new ManifestData(poolBuffer, 0, responseSize)); + + if (header.StoredAs == EChunkStorageFlags.None) + { + Unsafe.CopyBlockUnaligned(ref buffer[offset], ref poolBuffer[header.HeaderSize + chunkPartOffset], (uint)count); + if (!shouldCache) + return count; + using (var fileHandle = File.OpenHandle(cachePath!, FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.None, header.DataSizeCompressed)) + { + await RandomAccess.WriteAsync(fileHandle, new ReadOnlyMemory(poolBuffer, header.HeaderSize, header.DataSizeCompressed), 0, cancellationToken).ConfigureAwait(false); + RandomAccess.FlushToDisk(fileHandle); + } + CachePath = cachePath; + return count; + } + + if (header.StoredAs.HasFlag(EChunkStorageFlags.Encrypted)) + throw new NotSupportedException("Encrypted chunks are not supported"); + if (!header.StoredAs.HasFlag(EChunkStorageFlags.Compressed)) + throw new UnreachableException("Unknown/new chunk ChunkStorageFlag"); + if (manifest.Options.Decompressor is null) + throw new InvalidOperationException("Data is compressed and decompressor delegate was null"); + + // cant seek for uncompression + uncompressPoolBuffer = ArrayPool.Shared.Rent(header.DataSizeUncompressed); + + var result = manifest.Options.Decompressor.Invoke( + manifest.Options.DecompressorState, + poolBuffer, header.HeaderSize, header.DataSizeCompressed, + uncompressPoolBuffer, 0, header.DataSizeUncompressed); + if (!result) + throw new FileLoadException("Failed to uncompress data"); + + Unsafe.CopyBlockUnaligned(ref buffer[offset], ref uncompressPoolBuffer[chunkPartOffset], (uint)count); + if (!shouldCache) + return count; + using (var fileHandle = File.OpenHandle(cachePath!, FileMode.Create, FileAccess.Write, FileShare.None, FileOptions.None, header.DataSizeUncompressed)) + { + await RandomAccess.WriteAsync(fileHandle, new ReadOnlyMemory(uncompressPoolBuffer, 0, header.DataSizeUncompressed), 0, cancellationToken).ConfigureAwait(false); + RandomAccess.FlushToDisk(fileHandle); + } + CachePath = cachePath; + return count; + } + finally + { + if (poolBuffer is not null) + ArrayPool.Shared.Return(poolBuffer); + if (uncompressPoolBuffer is not null) + ArrayPool.Shared.Return(uncompressPoolBuffer); + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static void EnsureSuccessStatusCode(HttpResponseMessage res, Uri uri) + { + try + { + res.EnsureSuccessStatusCode(); + } + catch (HttpRequestException ex) + { + ex.Data.Add("Uri", uri); + ex.Data.Add("Headers", res.Headers); + throw; + } + } + + // ReSharper disable once UseSymbolAlias + internal static void Test_Zlibng(byte[] uncompressPoolBuffer, byte[] chunkBuffer, object zlibng, ManifestParseOptions.DecompressDelegate zlibngUncompress) + { + var header = FChunkHeader.Parse(chunkBuffer); + + var result = zlibngUncompress( + zlibng, + chunkBuffer, header.HeaderSize, header.DataSizeCompressed, + uncompressPoolBuffer, 0, header.DataSizeUncompressed); + + if (!result) + throw new FileLoadException("Failed to uncompress chunk data"); + } + + // ReSharper disable once UseSymbolAlias + internal static void Test_ZlibStream(byte[] uncompressPoolBuffer, byte[] chunkBuffer) + { + var header = FChunkHeader.Parse(chunkBuffer); + + var result = ManifestZlibStreamDecompressor.Decompress( + null, + chunkBuffer, header.HeaderSize, header.DataSizeCompressed, + uncompressPoolBuffer, 0, header.DataSizeUncompressed); + + if (!result) + throw new FileLoadException("Failed to uncompress chunk data"); + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkPart.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkPart.cs new file mode 100644 index 0000000..a54a9af --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FChunkPart.cs @@ -0,0 +1,49 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FChunkPart struct +/// +public readonly struct FChunkPart +{ + /// + /// The GUID of the chunk containing this part. + /// + public FGuid Guid { get; } + /// + /// The offset of the first byte into the chunk. + /// + public uint32 Offset { get; } + /// + /// The size of this part. + /// + public uint32 Size { get; } + + internal FChunkPart(FGuid guid, uint32 offset, uint32 size) + { + Guid = guid; + Offset = offset; + Size = size; + } + + internal FChunkPart(ref ManifestReader reader) + { + var startPos = reader.Position; + var dataSize = reader.Read(); + + Guid = reader.Read(); + Offset = reader.Read(); + Size = reader.Read(); + + reader.Position = startPos + dataSize; + } + +#if NET9_0_OR_GREATER + internal static FChunkPart Read(ref ManifestReader reader) => new(ref reader); +#else + internal static FChunkPart Read(GenericReader.IGenericReader genericReader) + { + var reader = (ManifestReader)genericReader; + return new FChunkPart(ref reader); + } +#endif +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FCustomField.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FCustomField.cs new file mode 100644 index 0000000..e015187 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FCustomField.cs @@ -0,0 +1,49 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FCustomField struct +/// +public sealed class FCustomField +{ + /// + /// Field name + /// + public string Name { get; internal set; } = ""; + /// + /// Field value + /// + public string Value { get; internal set; } = ""; + + internal FCustomField() { } + + internal static FCustomField[] ReadCustomFields(ref ManifestReader reader) + { + var startPos = reader.Position; + var dataSize = reader.Read(); + var dataVersion = reader.Read(); + var elementCount = reader.Read(); + + var fields = new FCustomField[elementCount]; + var fieldsSpan = fields.AsSpan(); + + if (dataVersion >= EChunkDataListVersion.Original) + { + for (var i = 0; i < elementCount; i++) + { + var field = new FCustomField(); + field.Name = reader.ReadFString(); + fieldsSpan[i] = field; + } + for (var i = 0; i < elementCount; i++) + fieldsSpan[i].Value = reader.ReadFString(); + } + else + { + var defaultField = new FCustomField(); + fieldsSpan.Fill(defaultField); + } + + reader.Position = startPos + dataSize; + return fields; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifest.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifest.cs new file mode 100644 index 0000000..b213dc2 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifest.cs @@ -0,0 +1,164 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FFileManifest struct +/// +public sealed class FFileManifest : IComparable, IComparable +{ + /// + /// The build relative filename. + /// + public string FileName { get; internal set; } = ""; + /// + /// Whether this is a symlink to another file. + /// + public string SymlinkTarget { get; internal set; } = ""; + /// + /// The file SHA1. + /// + public FSHAHash FileHash { get; internal set; } + /// + /// The flags for this file. + /// + public EFileMetaFlags FileMetaFlags { get; internal set; } + /// + /// The install tags for this file. + /// + public IReadOnlyList InstallTags { get; internal set; } = []; + /// + /// The list of chunk parts to stitch. + /// + public IReadOnlyList ChunkParts => ChunkPartsArray; + internal FChunkPart[] ChunkPartsArray = []; + /// + /// The size of this file. + /// + public int64 FileSize { get; internal set; } + /// + /// The mime type. + /// + public string MimeType { get; internal set; } = ""; + + internal FBuildPatchAppManifest Manifest { get; set; } = null!; + + internal FFileManifest() { } + + internal static FFileManifest[] ReadFileDataList(ref ManifestReader reader, FBuildPatchAppManifest manifest) + { + var startPos = reader.Position; + var dataSize = reader.Read(); + var dataVersion = reader.Read(); + var elementCount = reader.Read(); + + var files = new FFileManifest[elementCount]; + var filesSpan = files.AsSpan(); + + if (dataVersion >= EFileManifestListVersion.Original) + { + for (var i = 0; i < elementCount; i++) + { + var file = new FFileManifest(); + file.FileName = reader.ReadFString(); + filesSpan[i] = file; + } + for (var i = 0; i < elementCount; i++) + filesSpan[i].SymlinkTarget = reader.ReadFString(); + for (var i = 0; i < elementCount; i++) + filesSpan[i].FileHash = reader.Read(); + for (var i = 0; i < elementCount; i++) + filesSpan[i].FileMetaFlags = reader.Read(); + for (var i = 0; i < elementCount; i++) + filesSpan[i].InstallTags = reader.ReadFStringArray(); + for (var i = 0; i < elementCount; i++) + filesSpan[i].ChunkPartsArray = reader.ReadArray(FChunkPart.Read); + + // not to be found in UE, maybe fn specific? + if (dataVersion >= (EFileManifestListVersion)2) + { + for (var i = 0; i < elementCount; i++) // TArray + { + var a = reader.Read(); + reader.Position += a * 16; + } + for (var i = 0; i < elementCount; i++) + filesSpan[i]!.MimeType = reader.ReadFString(); + for (var i = 0; i < elementCount; i++) // Unknown + reader.Position += 32; + } + + // FileDataList.OnPostLoad(); + { + Array.Sort(files); + for (var i = 0; i < elementCount; i++) + { + var file = filesSpan[i]; + file.Manifest = manifest; + foreach (var chunkPart in file.ChunkPartsArray.AsSpan()) + { + file.FileSize += chunkPart.Size; + } + } + } + } + else + { + var defaultFile = new FFileManifest(); + filesSpan.Fill(defaultFile); + } + + reader.Position = startPos + dataSize; + return files; + } + + /// + /// Creates a read-only stream to read filedata from. + /// + public FFileManifestStream GetStream() => new(this, Manifest.Options.CacheChunksAsIs); + + /// + /// Creates a read-only stream to read filedata from. + /// + /// Whether or not to cache the chunks 1:1 as they were downloaded. + public FFileManifestStream GetStream(bool cacheAsIs) => new(this, cacheAsIs); + + + /// + public int CompareTo(FFileManifest? other) + { + if (ReferenceEquals(this, other)) return 0; + if (ReferenceEquals(null, other)) return 1; + return string.Compare(FileName, other.FileName, StringComparison.Ordinal); + } + + /// + public int CompareTo(object? obj) + { + if (ReferenceEquals(null, obj)) return 1; + if (ReferenceEquals(this, obj)) return 0; + return obj is FFileManifest other ? CompareTo(other) : throw new ArgumentException($"Object must be of type {nameof(FFileManifest)}"); + } + + /// + public static bool operator <(FFileManifest? left, FFileManifest? right) + { + return Comparer.Default.Compare(left, right) < 0; + } + + /// + public static bool operator >(FFileManifest? left, FFileManifest? right) + { + return Comparer.Default.Compare(left, right) > 0; + } + + /// + public static bool operator <=(FFileManifest? left, FFileManifest? right) + { + return Comparer.Default.Compare(left, right) <= 0; + } + + /// + public static bool operator >=(FFileManifest? left, FFileManifest? right) + { + return Comparer.Default.Compare(left, right) >= 0; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifestStream.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifestStream.cs new file mode 100644 index 0000000..85c1958 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FFileManifestStream.cs @@ -0,0 +1,662 @@ +using Microsoft.Win32.SafeHandles; +using OffiUtils; +using System.Buffers; +using System.Collections; +using System.Runtime.CompilerServices; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; +// ReSharper disable UseSymbolAlias + +/// +/// A stream representing a +/// +public sealed class FFileManifestStream : RandomAccessStream +{ + private readonly FFileManifest _fileManifest; + private readonly bool _cacheAsIs; + + /// Always + public override bool CanRead => true; + /// Always + public override bool CanSeek => true; + /// Always + public override bool CanWrite => false; + /// Gets the length/size of the stream + public override long Length => _fileManifest.FileSize; + + private long _position; + + /// Gets or sets the current position within the stream. + public override long Position + { + get => _position; + set + { + if ((ulong)value > (ulong)Length) + throw new ArgumentOutOfRangeException(nameof(Position), value, "Value is negative or exceeds the stream's length"); + + _position = value; + } + } + + /// Gets the file name of the represented by this stream + public string FileName => _fileManifest.FileName; + + internal FFileManifestStream(FFileManifest fileManifest, bool cacheAsIs) + { + if (string.IsNullOrEmpty(fileManifest.Manifest.Options.ChunkBaseUrl)) + throw new ArgumentException("Missing ChunkBaseUrl"); + if (fileManifest.Manifest.Meta.bIsFileData) + throw new NotSupportedException("File-data manifests are not supported"); + + _fileManifest = fileManifest; + _cacheAsIs = cacheAsIs; + } + + /// + /// Asynchronously saves the current stream to another stream. + /// + /// The destination stream. + /// The progress change callback. (optional) + /// The user state for the . (optional) + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public async Task SaveToAsync(Stream destination, Action? progressCallback, + object? userState = default, int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + if (destination is MemoryStream { Position: 0 } ms) + { + ms.Capacity = (int)Length; + if (ms.TryGetBuffer(out var buffer)) + { + await SaveBytesAsync(buffer.Array!, progressCallback, userState, maxDegreeOfParallelism, cancellationToken).ConfigureAwait(false); + ms.Position = (int)Length; + return; + } + } + + // TODO: make concurrent + + var downloadState = new DownloadState(null!, _fileManifest, Length, userState, progressCallback); + + if (_cacheAsIs) + { + var poolBuffer = ArrayPool.Shared.Rent(_fileManifest.Manifest.Options.ChunkDownloadBufferSize); + + try + { + foreach (var fileChunkPart in _fileManifest.ChunkPartsArray) + { + var chunk = _fileManifest.Manifest.Chunks[fileChunkPart.Guid]; + await chunk.ReadDataAsIsAsync(poolBuffer, _fileManifest.Manifest, cancellationToken).ConfigureAwait(false); + await destination.WriteAsync(new ReadOnlyMemory(poolBuffer, (int)fileChunkPart.Offset, (int)fileChunkPart.Size), + cancellationToken).ConfigureAwait(false); + downloadState.OnBytesWritten(fileChunkPart.Size); + } + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + else + { + var poolBuffer = ArrayPool.Shared.Rent(_fileManifest.Manifest.Options.ChunkDownloadBufferSize); + + try + { + foreach (var fileChunkPart in _fileManifest.ChunkPartsArray) + { + var chunk = _fileManifest.Manifest.Chunks[fileChunkPart.Guid]; + await chunk.ReadDataAsync(poolBuffer, 0, (int)fileChunkPart.Size, + (int)fileChunkPart.Offset, _fileManifest.Manifest, cancellationToken).ConfigureAwait(false); + await destination.WriteAsync(new ReadOnlyMemory(poolBuffer, 0, (int)fileChunkPart.Size), + cancellationToken).ConfigureAwait(false); + downloadState.OnBytesWritten(fileChunkPart.Size); + } + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + + await destination.FlushAsync(cancellationToken).ConfigureAwait(false); + } + + /// + public Task SaveToAsync(Stream destination, int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + return SaveToAsync(destination, null, 0, maxDegreeOfParallelism, cancellationToken); + } + + /// + /// Asynchronously saves the current stream to a buffer. + /// + /// The destination buffer. + /// The progress change callback. (optional) + /// The user state for the . (optional) + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public async Task SaveBytesAsync(byte[] destination, Action? progressCallback, + object? userState = default, int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + ArgumentOutOfRangeException.ThrowIfLessThan(destination.Length, Length); + + var downloadState = new DownloadState(destination, _fileManifest, Length, userState, progressCallback); + var parallelOptions = new ParallelOptions + { + MaxDegreeOfParallelism = maxDegreeOfParallelism ?? Environment.ProcessorCount, + CancellationToken = cancellationToken + }; + + if (_cacheAsIs) + await Parallel.ForEachAsync(downloadState, parallelOptions, SaveAsIsAsync).ConfigureAwait(false); + else + await Parallel.ForEachAsync(downloadState, parallelOptions, SaveAsync).ConfigureAwait(false); + + return; + + static async ValueTask SaveAsync(ChunkWithOffset tuple, CancellationToken token) + { + await tuple.Chunk.ReadDataAsync(tuple.State.Destination, (int)tuple.Offset, (int)tuple.ChunkPartSize, + (int)tuple.ChunkPartOffset, tuple.State.FileManifest.Manifest, token).ConfigureAwait(false); + tuple.State.OnBytesWritten(tuple.ChunkPartSize); + } + + static async ValueTask SaveAsIsAsync(ChunkWithOffset tuple, CancellationToken token) + { + var poolBuffer = ArrayPool.Shared.Rent(tuple.State.FileManifest.Manifest.Options.ChunkDownloadBufferSize); + + try + { + await tuple.Chunk.ReadDataAsIsAsync(poolBuffer, tuple.State.FileManifest.Manifest, token).ConfigureAwait(false); + Unsafe.CopyBlockUnaligned(ref tuple.State.Destination[tuple.Offset], + ref poolBuffer[tuple.ChunkPartOffset], tuple.ChunkPartSize); + tuple.State.OnBytesWritten(tuple.ChunkPartSize); + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + } + + /// + /// Asynchronously saves the current stream to a buffer. + /// + /// The destination buffer. + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public Task SaveBytesAsync(byte[] destination, int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + return SaveBytesAsync(destination, null, 0, maxDegreeOfParallelism, cancellationToken); + } + + /// + /// Asynchronously saves the current stream to a buffer. + /// + /// The progress change callback. (optional) + /// The user state for the . (optional) + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public async Task SaveBytesAsync(Action progressCallback, object? userState = default, + int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + var destination = new byte[Length]; + await SaveBytesAsync(destination, progressCallback, userState, maxDegreeOfParallelism, cancellationToken).ConfigureAwait(false); + return destination; + } + + /// + /// Asynchronously saves the current stream to a buffer. + /// + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public async Task SaveBytesAsync(int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + var destination = new byte[Length]; + await SaveBytesAsync(destination, maxDegreeOfParallelism, cancellationToken).ConfigureAwait(false); + return destination; + } + + /// + /// Asynchronously saves the current stream to a file. + /// + /// The path of the destination file. + /// The progress change callback. (optional) + /// The user state for the . (optional) + /// The maximum number of concurrent tasks saving/downloading to the destination. (optional) + /// The token to monitor for cancellation requests. + /// A task that represents the entire save operation. + public async Task SaveFileAsync(string path, Action? progressCallback, object? userState = null, + int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + using var destination = File.OpenHandle(path, FileMode.Create, FileAccess.ReadWrite, FileShare.None, FileOptions.Asynchronous, Length); + var downloadState = new DownloadState(destination, _fileManifest, Length, userState, progressCallback); + + var parallelOptions = new ParallelOptions + { + MaxDegreeOfParallelism = maxDegreeOfParallelism ?? Environment.ProcessorCount, + CancellationToken = cancellationToken + }; + + if (_cacheAsIs) + await Parallel.ForEachAsync(downloadState, parallelOptions, SaveAsIsAsync).ConfigureAwait(false); + else + await Parallel.ForEachAsync(downloadState, parallelOptions, SaveAsync).ConfigureAwait(false); + + RandomAccess.FlushToDisk(destination); + return; + + static async ValueTask SaveAsync(ChunkWithOffset tuple, CancellationToken token) + { + var poolBuffer = ArrayPool.Shared.Rent((int)tuple.ChunkPartSize); + + try + { + await tuple.Chunk.ReadDataAsync(poolBuffer, 0, (int)tuple.ChunkPartSize, + (int)tuple.ChunkPartOffset, tuple.State.FileManifest.Manifest, token).ConfigureAwait(false); + await RandomAccess.WriteAsync(tuple.State.Destination, + new ReadOnlyMemory(poolBuffer, 0, (int)tuple.ChunkPartSize), + tuple.Offset, token).ConfigureAwait(false); + tuple.State.OnBytesWritten(tuple.ChunkPartSize); + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + + static async ValueTask SaveAsIsAsync(ChunkWithOffset tuple, CancellationToken token) + { + var poolBuffer = ArrayPool.Shared.Rent(tuple.State.FileManifest.Manifest.Options.ChunkDownloadBufferSize); + + try + { + await tuple.Chunk.ReadDataAsIsAsync(poolBuffer, tuple.State.FileManifest.Manifest, token).ConfigureAwait(false); + await RandomAccess.WriteAsync(tuple.State.Destination, + new ReadOnlyMemory(poolBuffer, (int)tuple.ChunkPartOffset, (int)tuple.ChunkPartSize), + tuple.Offset, token).ConfigureAwait(false); + tuple.State.OnBytesWritten(tuple.ChunkPartSize); + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + } + + /// + public Task SaveFileAsync(string path, int? maxDegreeOfParallelism = null, CancellationToken cancellationToken = default) + { + return SaveFileAsync(path, null, 0, maxDegreeOfParallelism, cancellationToken); + } + + /// + /// Reads a sequence of bytes from the current stream. + /// + /// + /// When this method returns, contains the specified byte array with the values between + /// and ( + - 1) + /// replaced by the characters read from the current stream. + /// + /// The zero-based byte offset in at which to begin storing data from the current stream. + /// The maximum number of bytes to read. + /// The total number of bytes written into the . + public override int Read(byte[] buffer, int offset, int count) + { + return ReadAsync(buffer, offset, count, CancellationToken.None).GetAwaiter().GetResult(); + } + + /// + /// Asynchronously reads a sequence of bytes from the the current stream. + /// + /// + /// When this method returns, contains the specified byte array with the values between + /// and ( + - 1) + /// replaced by the characters read from the current stream. + /// + /// The zero-based byte offset in at which to begin storing data from the current stream. + /// The maximum number of bytes to read. + /// The token to monitor for cancellation requests. + /// The total number of bytes written into the . + public override async Task ReadAsync(byte[] buffer, int offset, int count, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + return await Task.FromCanceled(cancellationToken).ConfigureAwait(false); + + var bytesRead = await ReadAtAsync(_position, buffer, offset, count, cancellationToken).ConfigureAwait(false); + _position += bytesRead; + return bytesRead; + } + + /// + /// Asynchronously reads a sequence of bytes from the current stream. + /// + /// The region of memory to write the data into. + /// The token to monitor for cancellation requests. + /// The total number of bytes written into the . + public override async ValueTask ReadAsync(Memory buffer, CancellationToken cancellationToken = default) + { + var bytesRead = await ReadAtAsync(_position, buffer, cancellationToken).ConfigureAwait(false); + _position += bytesRead; + return bytesRead; + } + + /// + /// Reads a sequence of bytes from the given of the current stream. + /// + /// The position to begin reading from. + /// + /// When this method returns, contains the specified byte array with the values between + /// and ( + - 1) + /// replaced by the characters read from the current stream. + /// + /// The zero-based byte offset in at which to begin storing data from the current stream. + /// The maximum number of bytes to read. + /// The total number of bytes written into the . + public override int ReadAt(long position, byte[] buffer, int offset, int count) + { + return ReadAtAsync(position, buffer, offset, count, CancellationToken.None).GetAwaiter().GetResult(); + } + + /// + /// Asynchronously reads a sequence of bytes from the given of the current stream. + /// + /// The position to begin reading from. + /// + /// When this method returns, contains the specified byte array with the values between + /// and ( + - 1) + /// replaced by the characters read from the current stream. + /// + /// The zero-based byte offset in at which to begin storing data from the current stream. + /// The maximum number of bytes to read. + /// The token to monitor for cancellation requests. + /// The total number of bytes written into the . + public override async Task ReadAtAsync(long position, byte[] buffer, int offset, int count, CancellationToken cancellationToken = default) + { + var (i, startPos) = GetChunkIndex(position); + if (i == -1) + return 0; + + var bytesRead = 0u; + + if (_cacheAsIs) + { + var poolBuffer = ArrayPool.Shared.Rent(_fileManifest.Manifest.Options.ChunkDownloadBufferSize); + + try + { + while (i < _fileManifest.ChunkPartsArray.Length) + { + var chunkPart = _fileManifest.ChunkPartsArray[i]; + var chunk = _fileManifest.Manifest.Chunks[chunkPart.Guid]; + + await chunk.ReadDataAsIsAsync(poolBuffer, _fileManifest.Manifest, cancellationToken).ConfigureAwait(false); + + var chunkOffset = chunkPart.Offset + startPos; + var chunkBytes = chunkPart.Size - startPos; + var bytesLeft = (uint)count - bytesRead; + + if (bytesLeft <= chunkBytes) + { + Unsafe.CopyBlockUnaligned(ref buffer[bytesRead + offset], ref poolBuffer[chunkOffset], bytesLeft); + bytesRead += bytesLeft; + break; + } + + Unsafe.CopyBlockUnaligned(ref buffer[bytesRead + offset], ref poolBuffer[chunkOffset], chunkBytes); + bytesRead += chunkBytes; + startPos = 0; + + ++i; + } + } + finally + { + ArrayPool.Shared.Return(poolBuffer); + } + } + else + { + while (i < _fileManifest.ChunkPartsArray.Length) + { + var chunkPart = _fileManifest.ChunkPartsArray[i]; + var chunk = _fileManifest.Manifest.Chunks[chunkPart.Guid]; + + var chunkOffset = (int)(chunkPart.Offset + startPos); + var chunkBytes = (int)(chunkPart.Size - startPos); + var bytesLeft = count - (int)bytesRead; + + if (bytesLeft <= chunkBytes) + { + await chunk.ReadDataAsync(buffer, (int)bytesRead + offset, bytesLeft, chunkOffset, _fileManifest.Manifest, cancellationToken).ConfigureAwait(false); + bytesRead += (uint)bytesLeft; + break; + } + + await chunk.ReadDataAsync(buffer, (int)bytesRead + offset, chunkBytes, chunkOffset, _fileManifest.Manifest, cancellationToken).ConfigureAwait(false); + bytesRead += (uint)chunkBytes; + startPos = 0; + + ++i; + } + } + + return (int)bytesRead; + } + + private long _lastChunkPartPosition; + private uint _lastChunkPartSize; + private int _lastChunkPartIndex; + + private (int Index, uint ChunkPos) GetChunkIndexNew(long position) + { + lock (_fileManifest) + { + var maxPosition = _lastChunkPartPosition + _lastChunkPartSize; + if (maxPosition < position && position >= _lastChunkPartPosition) + { + return (_lastChunkPartIndex, (uint)(_lastChunkPartPosition - position)); + } + + var chunkPartPosition = 0L; + + for (var i = 0; i < _fileManifest.ChunkPartsArray.Length; i++) + { + var chunkPart = _fileManifest.ChunkPartsArray[i]; + + if (chunkPartPosition >= position) + { + _lastChunkPartPosition = chunkPartPosition; + _lastChunkPartSize = chunkPart.Size; + _lastChunkPartIndex = i; + return (i, (uint)(chunkPartPosition - position)); + } + + chunkPartPosition += chunkPart.Size; + } + + return (-1, 0); + } + } + + private (int Index, uint ChunkPos) GetChunkIndex(long position) + { + for (var i = 0; i < _fileManifest.ChunkPartsArray.Length; i++) + { + var chunkPart = _fileManifest.ChunkPartsArray[i]; + + if (position < chunkPart.Size) + return (i, (uint)position); + + position -= chunkPart.Size; + } + + return (-1, 0); + } + + /// Sets the position within the current stream to the specified value. + /// The new position within the stream. This is relative to the parameter, and can be positive or negative. + /// A value of type , which acts as the seek reference point. + /// The new position within the stream, calculated by combining the initial reference point and the offset. + /// There is an invalid . + public override long Seek(long offset, SeekOrigin loc) + { + Position = loc switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => offset + _position, + SeekOrigin.End => Length + offset, + _ => throw new ArgumentException("Invalid loc", nameof(loc)) + }; + return _position; + } + + /// Not supported + public override void SetLength(long value) + => throw new NotSupportedException(); + /// Not supported + public override void Write(byte[] buffer, int offset, int count) + => throw new NotSupportedException(); + /// Not supported + public override void Flush() + => throw new NotSupportedException(); +} + +/// +/// Event for save progress +/// +public sealed class SaveProgressChangedEventArgs : EventArgs +{ + internal SaveProgressChangedEventArgs(object? userState, long bytesSaved, long totalBytesToSave, int progressPercentage) + { + UserState = userState; + BytesSaved = bytesSaved; + TotalBytesToSave = totalBytesToSave; + ProgressPercentage = progressPercentage; + } + + /// + public object? UserState { get; } + /// + public long BytesSaved { get; } + /// + public long TotalBytesToSave { get; } + /// + public int ProgressPercentage { get; } +} + +internal sealed class DownloadState : IEnumerable>, IEnumerator> + where TDestination : class +{ + public readonly TDestination Destination; + public readonly FFileManifest FileManifest; + + private readonly LockObject? _lock; + private readonly object? _userState; + private readonly Action? _callback; + private readonly long _totalBytesToSave; + private long _bytesSaved; + private int _lastProgress; + + // IEnumerable & IEnumerator + private long _offset; + private long _lastSize; + private int _chunkpartIndex; + + public DownloadState(TDestination destination, FFileManifest fileManifest, long totalBytesToSave, object? userState, Action? callback) + { + Reset(); + Destination = destination; + FileManifest = fileManifest; + + if (callback is null) return; + _lock = new LockObject(); + _userState = userState; + _callback = callback; + _totalBytesToSave = totalBytesToSave; + } + + public void OnBytesWritten(long amount) + { + if (_callback is null) + return; + + lock (_lock!) + { + _bytesSaved += amount; + var progress = (int)MathF.Truncate((float)_bytesSaved / _totalBytesToSave * 100f); + if (progress != _lastProgress) + { + _lastProgress = progress; + var eventArgs = new SaveProgressChangedEventArgs(_userState, _bytesSaved, _totalBytesToSave, progress); + _callback(eventArgs); + } + } + } + + // IEnumerable + + public IEnumerator> GetEnumerator() => this; + IEnumerator IEnumerable.GetEnumerator() => this; + + // IEnumerator + + public bool MoveNext() + { + _chunkpartIndex++; + if (_chunkpartIndex >= FileManifest.ChunkPartsArray.Length) + return false; + + _offset += _lastSize; + var chunkPart = FileManifest.ChunkPartsArray[_chunkpartIndex]; + _lastSize = chunkPart.Size; + return true; + } + + public void Reset() + { + _offset = 0; + _lastSize = 0; + _chunkpartIndex = -1; + } + + public ChunkWithOffset Current + { + get + { + var chunkPart = FileManifest.ChunkPartsArray[_chunkpartIndex]; + var chunk = FileManifest.Manifest.Chunks[chunkPart.Guid]; + return new ChunkWithOffset(this, chunk, chunkPart.Offset, chunkPart.Size, _offset); + } + } + + object IEnumerator.Current => Current; + + public void Dispose() { } +} + +internal readonly struct ChunkWithOffset where TDestination : class +{ + public readonly DownloadState State; + public readonly FChunkInfo Chunk; + public readonly uint ChunkPartOffset; + public readonly uint ChunkPartSize; + public readonly long Offset; + + public ChunkWithOffset(DownloadState state, FChunkInfo chunk, uint chunkPartOffset, uint chunkPartSize, long offset) + { + State = state; + Chunk = chunk; + ChunkPartOffset = chunkPartOffset; + ChunkPartSize = chunkPartSize; + Offset = offset; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FGuid.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FGuid.cs new file mode 100644 index 0000000..1ca9a4d --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FGuid.cs @@ -0,0 +1,196 @@ +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Unicode; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FGuid struct +/// +[StructLayout(LayoutKind.Sequential, Pack = 1)] +public readonly struct FGuid : IEquatable, ISpanFormattable, IUtf8SpanFormattable +{ + /// + /// The size of the FGuid/struct. + /// + public const int Size = sizeof(uint32) * 4; + + private readonly uint32 A; + private readonly uint32 B; + private readonly uint32 C; + private readonly uint32 D; + + /// + /// Creates a hex string of the guid. + /// + public string GetHexString(bool upperCase = true) => upperCase + ? $"{A:X8}{B:X8}{C:X8}{D:X8}" + : $"{A:x8}{B:x8}{C:x8}{D:x8}"; + + /// + /// Creates a string of the guid. + /// + public string GetGuidString() => $"{A:x8}-{B >> 16:x4}-{B & 0xffff:x4}-{C >> 16:x4}-{C & 0xffff:x4}{D:x8}"; + + /// + /// Creates a FGuid from values. + /// + /// A value. + /// B value. + /// C value. + /// D value. + public FGuid(uint32 a, uint32 b, uint32 c, uint32 d) + { + A = a; + B = b; + C = c; + D = d; + } + + /// + /// Parses a FGuid from a string. + /// + /// The FGuid string. + public FGuid(string guid) : this(guid.AsSpan()) { } + + /// + /// Parses a FGuid from a string. + /// + /// The FGuid string. + public FGuid(ReadOnlySpan guid) + { + if (guid.Length != 32) + throw new ArgumentOutOfRangeException(nameof(guid), "guid has to be 32 characters long, other parsing is not implemented"); + A = uint32.Parse(guid[..8], NumberStyles.AllowHexSpecifier); + B = uint32.Parse(guid[8..16], NumberStyles.AllowHexSpecifier); + C = uint32.Parse(guid[16..24], NumberStyles.AllowHexSpecifier); + D = uint32.Parse(guid[24..32], NumberStyles.AllowHexSpecifier); + } + + /// + /// Parses a FGuid from a UTF8 string. + /// + /// The UTF8 FGuid string. + public FGuid(ReadOnlySpan utf8Guid) + { + if (utf8Guid.Length != 32) + throw new ArgumentOutOfRangeException(nameof(utf8Guid), "guid has to be 32 characters long, other parsing is not implemented"); + A = uint32.Parse(utf8Guid[..8], NumberStyles.AllowHexSpecifier); + B = uint32.Parse(utf8Guid[8..16], NumberStyles.AllowHexSpecifier); + C = uint32.Parse(utf8Guid[16..24], NumberStyles.AllowHexSpecifier); + D = uint32.Parse(utf8Guid[24..32], NumberStyles.AllowHexSpecifier); + } + + /// + /// Creates a random FGuid. + /// + public static FGuid Random() + { + Unsafe.SkipInit(out FGuid result); + RandomNumberGenerator.Fill(result.GetSpan()); + return result; + } + + /// + /// Checks for validity. + /// + public bool IsValid() => (A | B | C | D) != 0; + + /// + /// Gets the data of the guid. + /// + public ReadOnlySpan AsSpan() => + MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref Unsafe.AsRef(in A)), Size); + + /// + /// Gets the values of the guid. + /// + public ReadOnlySpan AsIntSpan() => + MemoryMarshal.CreateReadOnlySpan(ref Unsafe.AsRef(in A), 4); + + internal Span GetSpan() => + MemoryMarshal.CreateSpan(ref Unsafe.As(ref Unsafe.AsRef(in A)), Size); + + /// + public bool Equals(FGuid other) + { + return A == other.A && B == other.B && C == other.C && D == other.D; + } + + /// + public override bool Equals(object? obj) + { + return obj is FGuid other && Equals(other); + } + + /// + public override int GetHashCode() + { + return HashCode.Combine(A, B, C, D); + } + + /// + public static bool operator ==(FGuid left, FGuid right) + { + return left.Equals(right); + } + + /// + public static bool operator !=(FGuid left, FGuid right) + { + return !left.Equals(right); + } + + /// + public override string ToString() => GetHexString(); + + /// + public string ToString(string? format, IFormatProvider? formatProvider) + { + FormattableString formattable = $"{A:X8}{B:X8}{C:X8}{D:X8}"; + return formattable.ToString(formatProvider); + } + + /// + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) + { + return destination.TryWrite(provider, $"{A:X8}{B:X8}{C:X8}{D:X8}", out charsWritten); + } + + /// + public bool TryFormat(Span destination, out int bytesWritten, ReadOnlySpan format, IFormatProvider? provider) + { + return Utf8.TryWrite(destination, provider, $"{A:X8}{B:X8}{C:X8}{D:X8}", out bytesWritten); + } +} + +/// +/// Converts from and to JSON. +/// +public sealed class FGuidConverter : JsonConverter +{ + /// + public override FGuid Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.ValueSpan.IsEmpty) return default; + return new FGuid(reader.ValueSpan); + } + + /// + public override void Write(Utf8JsonWriter writer, FGuid value, JsonSerializerOptions options) + { + Span guidUtf8 = stackalloc byte[FGuid.Size * 2]; + + if (value.TryFormat(guidUtf8, out _, default, null)) + { + writer.WriteStringValue(guidUtf8); + return; + } + + writer.WriteStringValue(string.Empty); + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestHeader.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestHeader.cs new file mode 100644 index 0000000..afcc577 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestHeader.cs @@ -0,0 +1,64 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +internal class FManifestHeader +{ + public const uint32 Magic = 0x44BEC00C; + + /// + /// The version of this header and manifest data format, driven by the feature level. + /// + public readonly EFeatureLevel Version; + /// + /// The size of this header. + /// + public readonly int32 HeaderSize; + /// + /// The size of this data compressed. + /// + public readonly int32 DataSizeCompressed; + /// + /// The size of this data uncompressed. + /// + public readonly int32 DataSizeUncompressed; + /// + /// How the chunk data is stored. + /// + public readonly EManifestStorageFlags StoredAs; + /// + /// The SHA1 hash for the manifest data that follows. + /// + public readonly FSHAHash SHAHash; + + internal FManifestHeader(ref ManifestReader reader) + { + var magic = reader.Read(); + if (magic != Magic) + throw new FileLoadException($"Invalid manifest header magic: 0x{magic:X}"); + + HeaderSize = reader.Read(); + DataSizeUncompressed = reader.Read(); + DataSizeCompressed = reader.Read(); + SHAHash = reader.Read(); + StoredAs = reader.Read(); + Version = HeaderSize > ManifestHeaderVersionSizes[(int32)EFeatureLevel.Original] + ? reader.Read() + : EFeatureLevel.StoredAsCompressedUClass; + + reader.SetPosition(HeaderSize); + } + + // The constant minimum sizes for each version of a header struct. Must be updated. + // If new member variables are added the version MUST be bumped and handled properly here, + // and these values must never change. + private static readonly uint32[] ManifestHeaderVersionSizes = + [ + // EFeatureLevel::Original is 37B (32b Magic, 32b HeaderSize, 32b DataSizeUncompressed, 32b DataSizeCompressed, 160b SHA1, 8b StoredAs) + // This remained the same all up to including EFeatureLevel::StoresPrerequisiteIds. + 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, 37, + // EFeatureLevel::StoredAsBinaryData is 41B, (296b Original, 32b Version). + // This remained the same all up to including EFeatureLevel::UsesBuildTimeGeneratedBuildId. + 41, 41, 41, 41, 41, + // Undocumented, but the latest version is still 41B + 41, 41, 41 + ]; +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestMeta.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestMeta.cs new file mode 100644 index 0000000..c428183 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FManifestMeta.cs @@ -0,0 +1,107 @@ +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; + +/// +/// UE FManifestMeta struct +/// +public sealed class FManifestMeta +{ + /// + /// The feature level support this build was created with, regardless of the serialised format. + /// + public EFeatureLevel FeatureLevel { get; internal set; } = EFeatureLevel.Invalid; + /// + /// Whether this is a legacy 'nochunks' build. + /// + public bool bIsFileData { get; internal set; } + /// + /// The app id provided at generation. + /// + public uint32 AppID { get; internal set; } + /// + /// The app name string provided at generation. + /// + public string AppName { get; internal set; } = ""; + /// + /// The build version string provided at generation. + /// + public string BuildVersion { get; internal set; } = ""; + /// + /// The file in this manifest designated the application executable of the build. + /// + public string LaunchExe { get; internal set; } = ""; + /// + /// The command line required when launching the application executable. + /// + public string LaunchCommand { get; internal set; } = ""; + /// + /// The set of prerequisite ids for dependencies that this build's prerequisite installer will apply. + /// + public string[] PrereqIds { get; internal set; } = []; + /// + /// A display string for the prerequisite provided at generation. + /// + public string PrereqName { get; internal set; } = ""; + /// + /// The file in this manifest designated the launch executable of the prerequisite installer. + /// + public string PrereqPath { get; internal set; } = ""; + /// + /// The command line required when launching the prerequisite installer. + /// + public string PrereqArgs { get; internal set; } = ""; + /// + /// A unique build id generated at original chunking time to identify an exact build. + /// + public string BuildId { get; internal set; } = ""; + + /// + /// Undocumented + /// + public string UninstallExe { get; internal set; } = ""; + /// + /// Undocumented + /// + public string UninstallCommand { get; internal set; } = ""; + + internal FManifestMeta() { } + internal FManifestMeta(ref ManifestReader reader) + { + var startPos = reader.Position; + var dataSize = reader.Read(); + var dataVersion = reader.Read(); + + if (dataVersion >= EManifestMetaVersion.Original) + { + FeatureLevel = reader.Read(); + bIsFileData = reader.Read() == 1; + AppID = reader.Read(); + AppName = reader.ReadFString(); + BuildVersion = reader.ReadFString(); + LaunchExe = reader.ReadFString(); + LaunchCommand = reader.ReadFString(); + PrereqIds = reader.ReadFStringArray(); + PrereqName = reader.ReadFString(); + PrereqPath = reader.ReadFString(); + PrereqArgs = reader.ReadFString(); + } + + BuildId = dataVersion >= EManifestMetaVersion.SerialisesBuildId + ? reader.ReadFString() + : GetBackwardsCompatibleBuildId(this); + + // not to be found in UE, maybe fn specific? + if (FeatureLevel > EFeatureLevel.UsesBuildTimeGeneratedBuildId) + { + UninstallExe = reader.ReadFString(); + UninstallCommand = reader.ReadFString(); + } + + reader.Position = startPos + dataSize; + } + + internal static string GetBackwardsCompatibleBuildId(in FManifestMeta meta) + { + // TODO: https://github.com/EpicGames/UnrealEngine/blob/a937fa584fbd6d69b7cf9c527907040c9dbf54fc/Engine/Source/Runtime/Online/BuildPatchServices/Private/BuildPatchUtil.cpp#L166 + return ""; + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/FSHAHash.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FSHAHash.cs new file mode 100644 index 0000000..b538b18 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/FSHAHash.cs @@ -0,0 +1,163 @@ +using OffiUtils; +using System.Globalization; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.UE; +// ReSharper disable InconsistentNaming +// ReSharper disable UseSymbolAlias + +/// +/// UE FSHAHash struct +/// +[StructLayout(LayoutKind.Sequential, Pack = 1, Size = Size)] +public readonly struct FSHAHash : IEquatable, ISpanFormattable +{ + /// + /// The size of the hash/struct. + /// + public const int Size = 20; + + private readonly long Hash_00_07; + private readonly long Hash_08_15; + private readonly int Hash_16_19; + + /// + public bool Equals(FSHAHash other) + { + return Hash_00_07 == other.Hash_00_07 && Hash_08_15 == other.Hash_08_15 && Hash_16_19 == other.Hash_16_19; + } + + /// + public override bool Equals(object? obj) + { + return obj is FSHAHash other && Equals(other); + } + + /// + public override int GetHashCode() + { + return HashCode.Combine(Hash_00_07, Hash_08_15, Hash_16_19); + } + + /// + public static bool operator ==(FSHAHash left, FSHAHash right) + { + return left.Equals(right); + } + + /// + public static bool operator !=(FSHAHash left, FSHAHash right) + { + return !left.Equals(right); + } + + /// + /// Gets the data of the hash. + /// + /// Hash data in a read-only span + public ReadOnlySpan AsSpan() => + MemoryMarshal.CreateReadOnlySpan(ref Unsafe.As(ref Unsafe.AsRef(in Hash_00_07)), Size); + + internal Span GetSpan() => + MemoryMarshal.CreateSpan(ref Unsafe.As(ref Unsafe.AsRef(in Hash_00_07)), Size); + + /// + /// Computes the hash of data using the SHA1 algorithm. + /// + /// The data to hash + /// The computed hash + public static FSHAHash Compute(ReadOnlySpan source) + { + Unsafe.SkipInit(out FSHAHash hash); + SHA1.TryHashData(source, hash.GetSpan(), out _); + return hash; + } + + /// Returns a representation of the current instance. + /// The value of this , represented as a series of uppercase hexadecimal digits. + public override string ToString() => StringUtils.BytesToHexUpper(AsSpan()); + + /// Returns a representation of the current instance. + /// Whether or not to return an uppercase string. + /// The value of this , represented as a series of hexadecimal digits. + public string ToString(bool upperCase) => upperCase + ? StringUtils.BytesToHexUpper(AsSpan()) + : StringUtils.BytesToHexLower(AsSpan()); + + /// Returns a representation of the current instance, according to the provided format specifier. + /// A read-only span containing the character representing one of the following specifiers that indicates the exact format to use when interpreting input:
+ /// "x" or "X".
+ /// When is or empty, "X" is used. + /// + /// Unused, pass a null reference. + /// The value of this , represented as a series of hexadecimal digits in the specified format. + /// If an invalid format is used. + public string ToString(string? format, IFormatProvider? formatProvider) + { + if (format is null || format.Length == 0 || format == "X") + return StringUtils.BytesToHexUpper(AsSpan()); + if (format == "x") + return StringUtils.BytesToHexLower(AsSpan()); + throw new FormatException("the provided format is not valid"); + } + + /// + /// Tries to format the current instance into the provided character span. + /// + /// The span in which to write the as a span of characters. + /// When this method returns, contains the number of characters written into the span. + /// A read-only span containing the character representing one of the following specifiers that indicates the exact format to use when interpreting input:
+ /// "x" or "X".
+ /// When is empty, "X" is used. + /// + /// Unused, pass a null reference. + /// if the formatting was successful; otherwise, . + /// If an invalid format is used. + public bool TryFormat(Span destination, out int charsWritten, ReadOnlySpan format, IFormatProvider? provider) + { + if (format.IsEmpty || format is "X") + return StringUtils.TryWriteBytesToHexUpper(AsSpan(), destination, out charsWritten); + if (format is "x") + return StringUtils.TryWriteBytesToHexLower(AsSpan(), destination, out charsWritten); + throw new FormatException("the provided format is not valid"); + } +} + +/// +/// Converts from and to JSON. +/// +public sealed class FSHAHashConverter : JsonConverter +{ + /// + public override FSHAHash Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + Unsafe.SkipInit(out FSHAHash result); + var resultSpan = result.GetSpan(); + var span = reader.ValueSpan; + + for (var i = 0; i < resultSpan.Length; i++) + { + resultSpan[i] = byte.Parse(span.Slice(i * 2, 2), NumberStyles.AllowHexSpecifier); + } + + return result; + } + + /// + public override void Write(Utf8JsonWriter writer, FSHAHash value, JsonSerializerOptions options) + { + Span hashUtf16 = stackalloc char[FSHAHash.Size * 2]; + + if (value.TryFormat(hashUtf16, out _, default, null)) + { + writer.WriteStringValue(hashUtf16); + return; + } + + writer.WriteStringValue(string.Empty); + } +} diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/UE/TypeAliases.cs b/DeveLanCacheUI_Backend.EpicManifestParser/UE/TypeAliases.cs new file mode 100644 index 0000000..afeadc0 --- /dev/null +++ b/DeveLanCacheUI_Backend.EpicManifestParser/UE/TypeAliases.cs @@ -0,0 +1,5 @@ +global using int32 = System.Int32; +global using int64 = System.Int64; +global using uint32 = System.UInt32; +global using uint64 = System.UInt64; +global using uint8 = System.Byte; diff --git a/DeveLanCacheUI_Backend.sln b/DeveLanCacheUI_Backend.sln index 6dcd8a1..788a2ba 100644 --- a/DeveLanCacheUI_Backend.sln +++ b/DeveLanCacheUI_Backend.sln @@ -7,6 +7,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DeveLanCacheUI_Backend", "D EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DeveLanCacheUI_Backend.Tests", "DeveLanCacheUI_Backend.Tests\DeveLanCacheUI_Backend.Tests.csproj", "{61B234FC-1790-4951-B763-9D93CA535DF8}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DeveLanCacheUI_Backend.EpicManifestParser", "DeveLanCacheUI_Backend.EpicManifestParser\DeveLanCacheUI_Backend.EpicManifestParser.csproj", "{8CBBAE99-5E56-41BA-BB81-66FD5E459631}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DeveLanCacheUI_Backend.EpicManifestParser.Tests", "DeveLanCacheUI_Backend.EpicManifestParser.Tests\DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj", "{78517F5E-180D-41D8-B9EE-84B8F3811F92}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -21,6 +25,14 @@ Global {61B234FC-1790-4951-B763-9D93CA535DF8}.Debug|Any CPU.Build.0 = Debug|Any CPU {61B234FC-1790-4951-B763-9D93CA535DF8}.Release|Any CPU.ActiveCfg = Release|Any CPU {61B234FC-1790-4951-B763-9D93CA535DF8}.Release|Any CPU.Build.0 = Release|Any CPU + {8CBBAE99-5E56-41BA-BB81-66FD5E459631}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8CBBAE99-5E56-41BA-BB81-66FD5E459631}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8CBBAE99-5E56-41BA-BB81-66FD5E459631}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8CBBAE99-5E56-41BA-BB81-66FD5E459631}.Release|Any CPU.Build.0 = Release|Any CPU + {78517F5E-180D-41D8-B9EE-84B8F3811F92}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {78517F5E-180D-41D8-B9EE-84B8F3811F92}.Debug|Any CPU.Build.0 = Debug|Any CPU + {78517F5E-180D-41D8-B9EE-84B8F3811F92}.Release|Any CPU.ActiveCfg = Release|Any CPU + {78517F5E-180D-41D8-B9EE-84B8F3811F92}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From e5d268e2ee430482c7719234f744cdd653484e6f Mon Sep 17 00:00:00 2001 From: Devedse Date: Tue, 6 May 2025 01:15:50 +0200 Subject: [PATCH 02/15] wip --- .../DbAsyncLogEntryProcessingQueueItem.cs | 10 ++ .../Db/DeveLanCacheUIDbContext.cs | 5 + .../Services/EpicManifestService.cs | 121 ++++++++++++++++++ 3 files changed, 136 insertions(+) create mode 100644 DeveLanCacheUI_Backend/Db/DbModels/DbAsyncLogEntryProcessingQueueItem.cs create mode 100644 DeveLanCacheUI_Backend/Services/EpicManifestService.cs diff --git a/DeveLanCacheUI_Backend/Db/DbModels/DbAsyncLogEntryProcessingQueueItem.cs b/DeveLanCacheUI_Backend/Db/DbModels/DbAsyncLogEntryProcessingQueueItem.cs new file mode 100644 index 0000000..b151902 --- /dev/null +++ b/DeveLanCacheUI_Backend/Db/DbModels/DbAsyncLogEntryProcessingQueueItem.cs @@ -0,0 +1,10 @@ +namespace DeveLanCacheUI_Backend.Db.DbModels +{ + public class DbAsyncLogEntryProcessingQueueItem + { + [Key] + public int Id { get; set; } + + public LanCacheLogEntryRaw LanCacheLogEntryRaw { get; set; } = null!; + } +} diff --git a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs index cde879a..0250fd6 100644 --- a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs +++ b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs @@ -6,6 +6,8 @@ public class DeveLanCacheUIDbContext : DbContext public DbSet DownloadEvents => Set(); public DbSet Settings => Set(); public DbSet SteamManifests => Set(); + public DbSet ManifestAsyncDownloadProcessingQueueItems => Set(); + public DeveLanCacheUIDbContext(DbContextOptions options) : base(options) { @@ -24,6 +26,9 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity() .HasKey(pc => new { pc.SteamDepotId, pc.SteamAppId }); + + modelBuilder.Entity() + .HasIndex(pc => pc.LanCacheLogEntryRaw); } } } diff --git a/DeveLanCacheUI_Backend/Services/EpicManifestService.cs b/DeveLanCacheUI_Backend/Services/EpicManifestService.cs new file mode 100644 index 0000000..30bb9df --- /dev/null +++ b/DeveLanCacheUI_Backend/Services/EpicManifestService.cs @@ -0,0 +1,121 @@ +namespace DeveLanCacheUI_Backend.Services +{ + public class EpicManifestService + { + private readonly ILogger _logger; + + public EpicManifestService(DeveLanCacheConfiguration deveLanCacheConfiguration, ILogger logger) + { + _logger = logger; + } + + public void TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) + { + //This method could use some TPL Dataflow, I now use locking which should be okayish + + if (!lanCacheLogEntryRaw.Request.Contains("/manifest/") || lanCacheLogEntryRaw.DownloadIdentifier == null) + { + _logger.LogError("Code bug: Trying to download manifest that isn't actually a manifest: {OriginalLogLine}", lanCacheLogEntryRaw.OriginalLogLine); + return; + } + + _ = Task.Run(async () => + { + var fallbackPolicy = Policy + .Handle() + .FallbackAsync(async (ct) => + { + await Task.CompletedTask; + _logger.LogInformation("Manifest saving: All retries failed, skipping..."); + }); + + var retryPolicy = Policy + .Handle() + .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), + (exception, timeSpan, context) => + { + _logger.LogInformation("Manifest saving: An error occurred while trying to save changes: {Message}", exception.Message); + }); + + await fallbackPolicy.WrapAsync(retryPolicy).ExecuteAsync(async () => + { + try + { + _semaphoreSlim.Wait(); + await using (var scope = _services.CreateAsyncScope()) + { + var theManifestUrlPart = lanCacheLogEntryRaw.Request.Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]; + + var everythingAfterManifest = theManifestUrlPart.Split("/manifest/", StringSplitOptions.RemoveEmptyEntries).Last(); + var manifestId = everythingAfterManifest.Split("/", StringSplitOptions.RemoveEmptyEntries).First(); + + //Replace invalid chars should dissalow reading any file you want :) + var manifestIdFileName = RemoveNonNumericCharacters(manifestId) + ".bin"; + var depotId = RemoveNonNumericCharacters(lanCacheLogEntryRaw.DownloadIdentifier!); + var depotIdAndManifestIdentifier = Path.Combine(depotId, manifestIdFileName); + + + using var dbContext = scope.ServiceProvider.GetRequiredService(); + var dbManifestFound = await dbContext.SteamManifests.FirstOrDefaultAsync(t => t.UniqueManifestIdentifier == depotIdAndManifestIdentifier); + + if (dbManifestFound != null) + { + return; + } + + var fullPath = Path.Combine(_manifestDirectory, depotIdAndManifestIdentifier); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + + + var cachedUrl = $"http://lancache.steamcontent.com{theManifestUrlPart}"; + using var httpClient = _httpClientFactoryForManifestDownloads.CreateClient(); + httpClient.DefaultRequestHeaders.Add("Host", lanCacheLogEntryRaw.Host); + httpClient.DefaultRequestHeaders.Add("User-Agent", "Valve/Steam HTTP Client 1.0"); + httpClient.DefaultRequestHeaders.Referrer = LanCacheLogReaderHostedService.SkipLogLineReferrer; //Add this to ensure we don't process this line again + var manifestResponse = await httpClient.GetAsync(cachedUrl); + + + + + + if (!manifestResponse.IsSuccessStatusCode) + { + _logger.LogWarning("Warning: Tried to obtain manifest for: {DownloadIdentifier} but status code was: {StatusCode}", lanCacheLogEntryRaw.DownloadIdentifier, manifestResponse.StatusCode); + return; + } + var manifestBytes = await manifestResponse.Content.ReadAsByteArrayAsync(); + + + var dbManifest = ManifestBytesToDbSteamManifest(manifestBytes, depotIdAndManifestIdentifier); + + if (dbManifest == null) + { + _logger.LogWarning("Could not get manifest for depot: {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); + return; + } + + var dbValue = dbContext.SteamManifests.FirstOrDefault(t => t.DepotId == dbManifest.DepotId && t.CreationTime == dbManifest.CreationTime); + if (dbValue != null) + { + dbContext.Entry(dbValue).CurrentValues.SetValues(dbManifest); + _logger.LogInformation("Updated manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); + } + else + { + await dbContext.SteamManifests.AddAsync(dbManifest); + _logger.LogInformation("Added manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); + } + + await File.WriteAllBytesAsync(fullPath, manifestBytes); + await dbContext.SaveChangesAsync(); + } + } + finally + { + _semaphoreSlim.Release(); + } + }); + }); + } + } +} From 14e43ab73e1558608915a692b177f8960ef4b052 Mon Sep 17 00:00:00 2001 From: Devedse Date: Tue, 6 May 2025 12:34:33 +0200 Subject: [PATCH 03/15] A bit of cleanup --- .../EpicManifestParserTests.cs | 20 +++---------- .../EpicManifestParser.cs | 30 ++++++++++++++----- 2 files changed, 27 insertions(+), 23 deletions(-) diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs index a889cf8..d4db29a 100644 --- a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs +++ b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs @@ -1,28 +1,16 @@ -using DeveLanCacheUI_Backend.EpicManifestParser.Decompressor; -using DeveLanCacheUI_Backend.EpicManifestParser.UE; - -namespace DeveLanCacheUI_Backend.EpicManifestParser.Tests +namespace DeveLanCacheUI_Backend.EpicManifestParser.Tests { [TestClass] - public sealed class EpicManifestParser + public sealed class EpicManifestParserTests { [TestMethod] public async Task DoesItWork() { // Arrange - var options = new ManifestParseOptions - { - //ChunkBaseUrl = "http://download.epicgames.com/Builds/UnrealEngineLauncher/CloudDir/", - //ChunkCacheDirectory = Directory.CreateDirectory(Path.Combine(Benchmarks.DownloadsDir, "chunks_v2")).FullName, - //ManifestCacheDirectory = Directory.CreateDirectory(Path.Combine(Benchmarks.DownloadsDir, "manifests_v2")).FullName, - }; - - options.Decompressor = ManifestZlibDotNetDecompressor.Decompress; - + var manifestBuffer = await File.ReadAllBytesAsync(Path.Combine("TestFiles", "1sE9O19OT_X-rOWTFEiMUGNBYu8I1A.manifest")); // Act - var manifestBuffer = await File.ReadAllBytesAsync(Path.Combine("TestFiles", "1sE9O19OT_X-rOWTFEiMUGNBYu8I1A.manifest")); - var manifest = FBuildPatchAppManifest.Deserialize(manifestBuffer, options); + var manifest = EpicManifestParser.Deserialize(manifestBuffer); // Assert Assert.AreEqual("Super Space Club.exe", manifest.Meta.LaunchExe); diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs b/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs index f621fa6..2942cf5 100644 --- a/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs +++ b/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs @@ -1,7 +1,23 @@ -//namespace DeveLanCacheUI_Backend.EpicManifestParser -//{ -// public class EpicManifestParser -// { -// public async Task<> -// } -//} +using DeveLanCacheUI_Backend.EpicManifestParser.Decompressor; +using DeveLanCacheUI_Backend.EpicManifestParser.UE; + +namespace DeveLanCacheUI_Backend.EpicManifestParser +{ + public class EpicManifestParser + { + public static FBuildPatchAppManifest Deserialize(ManifestRoData manifestBuffer) + { + var options = new ManifestParseOptions + { + //ChunkBaseUrl = "http://download.epicgames.com/Builds/UnrealEngineLauncher/CloudDir/", + //ChunkCacheDirectory = Directory.CreateDirectory(Path.Combine(Benchmarks.DownloadsDir, "chunks_v2")).FullName, + //ManifestCacheDirectory = Directory.CreateDirectory(Path.Combine(Benchmarks.DownloadsDir, "manifests_v2")).FullName, + }; + + options.Decompressor = ManifestZlibDotNetDecompressor.Decompress; + + var manifest = FBuildPatchAppManifest.Deserialize(manifestBuffer, options); + return manifest; + } + } +} From 75c894dd7d07f9d596824326a82d3de4ba7a04cf Mon Sep 17 00:00:00 2001 From: Devedse Date: Tue, 6 May 2025 15:32:13 +0200 Subject: [PATCH 04/15] WIP manifest parsing --- .../EpicManifestParser.cs | 2 +- .../Db/DeveLanCacheUIDbContext.cs | 2 +- .../DeveLanCacheUI_Backend.csproj | 4 + ...cessingQueueItemsProcessorHostedService.cs | 61 +++++++ .../LanCacheLogReaderHostedService.cs | 23 ++- .../Services/EpicManifestService.cs | 142 ++++++---------- .../Services/SteamManifestService.cs | 155 ++++++++---------- 7 files changed, 199 insertions(+), 190 deletions(-) create mode 100644 DeveLanCacheUI_Backend/LogReading/AsyncLogEntryProcessingQueueItemsProcessorHostedService.cs diff --git a/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs b/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs index 2942cf5..cc6c9f6 100644 --- a/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs +++ b/DeveLanCacheUI_Backend.EpicManifestParser/EpicManifestParser.cs @@ -3,7 +3,7 @@ namespace DeveLanCacheUI_Backend.EpicManifestParser { - public class EpicManifestParser + public static class EpicManifestParser { public static FBuildPatchAppManifest Deserialize(ManifestRoData manifestBuffer) { diff --git a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs index 0250fd6..582de4a 100644 --- a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs +++ b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs @@ -6,7 +6,7 @@ public class DeveLanCacheUIDbContext : DbContext public DbSet DownloadEvents => Set(); public DbSet Settings => Set(); public DbSet SteamManifests => Set(); - public DbSet ManifestAsyncDownloadProcessingQueueItems => Set(); + public DbSet AsyncLogEntryProcessingQueueItems => Set(); public DeveLanCacheUIDbContext(DbContextOptions options) : base(options) diff --git a/DeveLanCacheUI_Backend/DeveLanCacheUI_Backend.csproj b/DeveLanCacheUI_Backend/DeveLanCacheUI_Backend.csproj index 1ac4689..b7f0532 100644 --- a/DeveLanCacheUI_Backend/DeveLanCacheUI_Backend.csproj +++ b/DeveLanCacheUI_Backend/DeveLanCacheUI_Backend.csproj @@ -40,4 +40,8 @@ + + + + diff --git a/DeveLanCacheUI_Backend/LogReading/AsyncLogEntryProcessingQueueItemsProcessorHostedService.cs b/DeveLanCacheUI_Backend/LogReading/AsyncLogEntryProcessingQueueItemsProcessorHostedService.cs new file mode 100644 index 0000000..d62ae52 --- /dev/null +++ b/DeveLanCacheUI_Backend/LogReading/AsyncLogEntryProcessingQueueItemsProcessorHostedService.cs @@ -0,0 +1,61 @@ +using DeveLanCacheUI_Backend.Services; +using ProtoBuf.Meta; + +namespace DeveLanCacheUI_Backend.LogReading +{ + public class AsyncLogEntryProcessingQueueItemsProcessorHostedService : BackgroundService + { + private readonly IServiceProvider _services; + private readonly ILogger _logger; + + public AsyncLogEntryProcessingQueueItemsProcessorHostedService( + IServiceProvider services, + ILogger logger) + { + _services = services; + _logger = logger; + } + + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); + await GoRun(stoppingToken); + } + + private async Task GoRun(CancellationToken stoppingToken) + { + while (!stoppingToken.IsCancellationRequested) + { + await using (var scope = _services.CreateAsyncScope()) + { + using var dbContext = scope.ServiceProvider.GetRequiredService(); + + var items = await dbContext.AsyncLogEntryProcessingQueueItems.OrderBy(t => t.Id).ToListAsync(stoppingToken); + if (items.Count == 0) + { + await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); + continue; + } + + var steamManifestService = scope.ServiceProvider.GetRequiredService(); + var epicManifestService = scope.ServiceProvider.GetRequiredService(); + + foreach (var item in items) + { + if (item.LanCacheLogEntryRaw.CacheIdentifier == "steam") + { + await steamManifestService.TryToDownloadManifest(item.LanCacheLogEntryRaw); + } + if (item.LanCacheLogEntryRaw.CacheIdentifier == "epicgames") + { + await epicManifestService.TryToDownloadManifest(item.LanCacheLogEntryRaw); + } + } + + dbContext.AsyncLogEntryProcessingQueueItems.RemoveRange(items); + await dbContext.SaveChangesAsync(stoppingToken); + } + } + } + } +} diff --git a/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs b/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs index 1c24e89..ab34e64 100644 --- a/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs +++ b/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs @@ -9,7 +9,6 @@ public class LanCacheLogReaderHostedService : BackgroundService private readonly IServiceProvider _services; private readonly DeveLanCacheConfiguration _deveLanCacheConfiguration; - private readonly SteamManifestService _steamManifestService; private readonly ILogger _logger; /// @@ -43,12 +42,10 @@ public class LanCacheLogReaderHostedService : BackgroundService public LanCacheLogReaderHostedService(IServiceProvider services, DeveLanCacheConfiguration deveLanCacheConfiguration, - SteamManifestService steamManifestService, ILogger logger) { _services = services; _deveLanCacheConfiguration = deveLanCacheConfiguration; - _steamManifestService = steamManifestService; _logger = logger; } @@ -125,11 +122,23 @@ await retryPolicy.ExecuteAsync(async () => { continue; } - if (lanCacheLogLine.CacheIdentifier == "steam" && lanCacheLogLine.Request.Contains("/manifest/") && DateTime.Now < lanCacheLogLine.DateTime.AddDays(14)) + + if ( + (lanCacheLogLine.CacheIdentifier == "steam" && lanCacheLogLine.Request.Contains("/manifest/")) || + (lanCacheLogLine.CacheIdentifier == "epicgames" && lanCacheLogLine.Request.Contains(".manifest?")) + ) { - _logger.LogInformation("Found manifest for Depot: {DownloadIdentifier}", lanCacheLogLine.DownloadIdentifier); - var ttt = lanCacheLogLine; - _steamManifestService.TryToDownloadManifest(ttt); + if (DateTime.Now < lanCacheLogLine.DateTime.AddDays(14)) + { + // We found a manifest. This contains more information about the download. We'll process these asynchronously + // since it requires downloading and parsing the manifest file. + _logger.LogInformation("Found manifest for download: {DownloadIdentifier}", lanCacheLogLine.DownloadIdentifier); + var itemToProcessAsynchronously = new DbAsyncLogEntryProcessingQueueItem() + { + LanCacheLogEntryRaw = lanCacheLogLine + }; + dbContext.AsyncLogEntryProcessingQueueItems.Add(itemToProcessAsynchronously); + } } var cacheKey = $"{lanCacheLogLine.CacheIdentifier}_||_{lanCacheLogLine.DownloadIdentifier}_||_{lanCacheLogLine.RemoteAddress}"; diff --git a/DeveLanCacheUI_Backend/Services/EpicManifestService.cs b/DeveLanCacheUI_Backend/Services/EpicManifestService.cs index 30bb9df..58f56db 100644 --- a/DeveLanCacheUI_Backend/Services/EpicManifestService.cs +++ b/DeveLanCacheUI_Backend/Services/EpicManifestService.cs @@ -1,120 +1,74 @@ -namespace DeveLanCacheUI_Backend.Services + +namespace DeveLanCacheUI_Backend.Services { public class EpicManifestService { - private readonly ILogger _logger; - - public EpicManifestService(DeveLanCacheConfiguration deveLanCacheConfiguration, ILogger logger) + private readonly DeveLanCacheUIDbContext _dbContext; + private readonly IHttpClientFactory _httpClientFactoryForManifestDownloads; + private readonly ILogger _logger; + private readonly string _manifestDirectory; + + public EpicManifestService( + DeveLanCacheConfiguration deveLanCacheConfiguration, + DeveLanCacheUIDbContext dbContext, + IHttpClientFactory httpClientFactory, + ILogger logger) { + _dbContext = dbContext; + _httpClientFactoryForManifestDownloads = httpClientFactory; _logger = logger; + + var deveLanCacheUIDataDirectory = deveLanCacheConfiguration.DeveLanCacheUIDataDirectory ?? string.Empty; + _manifestDirectory = Path.Combine(deveLanCacheUIDataDirectory, "manifests"); } - public void TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) + public async Task TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) { - //This method could use some TPL Dataflow, I now use locking which should be okayish - if (!lanCacheLogEntryRaw.Request.Contains("/manifest/") || lanCacheLogEntryRaw.DownloadIdentifier == null) { _logger.LogError("Code bug: Trying to download manifest that isn't actually a manifest: {OriginalLogLine}", lanCacheLogEntryRaw.OriginalLogLine); return; } - _ = Task.Run(async () => - { - var fallbackPolicy = Policy - .Handle() - .FallbackAsync(async (ct) => - { - await Task.CompletedTask; - _logger.LogInformation("Manifest saving: All retries failed, skipping..."); - }); - - var retryPolicy = Policy - .Handle() - .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), - (exception, timeSpan, context) => - { - _logger.LogInformation("Manifest saving: An error occurred while trying to save changes: {Message}", exception.Message); - }); - - await fallbackPolicy.WrapAsync(retryPolicy).ExecuteAsync(async () => + var fallbackPolicy = Policy + .Handle() + .FallbackAsync(async (ct) => { - try - { - _semaphoreSlim.Wait(); - await using (var scope = _services.CreateAsyncScope()) - { - var theManifestUrlPart = lanCacheLogEntryRaw.Request.Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]; - - var everythingAfterManifest = theManifestUrlPart.Split("/manifest/", StringSplitOptions.RemoveEmptyEntries).Last(); - var manifestId = everythingAfterManifest.Split("/", StringSplitOptions.RemoveEmptyEntries).First(); - - //Replace invalid chars should dissalow reading any file you want :) - var manifestIdFileName = RemoveNonNumericCharacters(manifestId) + ".bin"; - var depotId = RemoveNonNumericCharacters(lanCacheLogEntryRaw.DownloadIdentifier!); - var depotIdAndManifestIdentifier = Path.Combine(depotId, manifestIdFileName); - - - using var dbContext = scope.ServiceProvider.GetRequiredService(); - var dbManifestFound = await dbContext.SteamManifests.FirstOrDefaultAsync(t => t.UniqueManifestIdentifier == depotIdAndManifestIdentifier); - - if (dbManifestFound != null) - { - return; - } - - var fullPath = Path.Combine(_manifestDirectory, depotIdAndManifestIdentifier); - Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); - - - var cachedUrl = $"http://lancache.steamcontent.com{theManifestUrlPart}"; - using var httpClient = _httpClientFactoryForManifestDownloads.CreateClient(); - httpClient.DefaultRequestHeaders.Add("Host", lanCacheLogEntryRaw.Host); - httpClient.DefaultRequestHeaders.Add("User-Agent", "Valve/Steam HTTP Client 1.0"); - httpClient.DefaultRequestHeaders.Referrer = LanCacheLogReaderHostedService.SkipLogLineReferrer; //Add this to ensure we don't process this line again - var manifestResponse = await httpClient.GetAsync(cachedUrl); - - + await Task.CompletedTask; + _logger.LogInformation("Manifest saving: All retries failed, skipping..."); + }); + var retryPolicy = Policy + .Handle() + .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), + (exception, timeSpan, context) => + { + _logger.LogInformation("Manifest saving: An error occurred while trying to save changes: {Message}", exception.Message); + }); + await fallbackPolicy.WrapAsync(retryPolicy).ExecuteAsync(async () => + { + var theManifestUrlPart = lanCacheLogEntryRaw.Request.Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]; - if (!manifestResponse.IsSuccessStatusCode) - { - _logger.LogWarning("Warning: Tried to obtain manifest for: {DownloadIdentifier} but status code was: {StatusCode}", lanCacheLogEntryRaw.DownloadIdentifier, manifestResponse.StatusCode); - return; - } - var manifestBytes = await manifestResponse.Content.ReadAsByteArrayAsync(); + using var httpClient = _httpClientFactoryForManifestDownloads.CreateClient(); + httpClient.DefaultRequestHeaders.Add("Host", lanCacheLogEntryRaw.Host); + httpClient.DefaultRequestHeaders.Add("User-Agent", "Valve/Steam HTTP Client 1.0"); + httpClient.DefaultRequestHeaders.Referrer = LanCacheLogReaderHostedService.SkipLogLineReferrer; //Add this to ensure we don't process this line again + var manifestResponse = await httpClient.GetAsync(theManifestUrlPart); - var dbManifest = ManifestBytesToDbSteamManifest(manifestBytes, depotIdAndManifestIdentifier); + if (!manifestResponse.IsSuccessStatusCode) + { + _logger.LogWarning("Warning: Tried to obtain manifest for: {DownloadIdentifier} but status code was: {StatusCode}", lanCacheLogEntryRaw.DownloadIdentifier, manifestResponse.StatusCode); + return; + } + var manifestBytes = await manifestResponse.Content.ReadAsByteArrayAsync(); - if (dbManifest == null) - { - _logger.LogWarning("Could not get manifest for depot: {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); - return; - } + var parsedManifest = EpicManifestParser.EpicManifestParser.Deserialize(manifestBytes); - var dbValue = dbContext.SteamManifests.FirstOrDefault(t => t.DepotId == dbManifest.DepotId && t.CreationTime == dbManifest.CreationTime); - if (dbValue != null) - { - dbContext.Entry(dbValue).CurrentValues.SetValues(dbManifest); - _logger.LogInformation("Updated manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); - } - else - { - await dbContext.SteamManifests.AddAsync(dbManifest); - _logger.LogInformation("Added manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); - } + _logger.LogInformation("Parsed epic manifest for Launch Exe: {Manifest}", parsedManifest.Meta.LaunchExe); - await File.WriteAllBytesAsync(fullPath, manifestBytes); - await dbContext.SaveChangesAsync(); - } - } - finally - { - _semaphoreSlim.Release(); - } - }); + // Save changes happens in the caller }); } } diff --git a/DeveLanCacheUI_Backend/Services/SteamManifestService.cs b/DeveLanCacheUI_Backend/Services/SteamManifestService.cs index e5a1522..4fbb6ce 100644 --- a/DeveLanCacheUI_Backend/Services/SteamManifestService.cs +++ b/DeveLanCacheUI_Backend/Services/SteamManifestService.cs @@ -2,16 +2,18 @@ { public class SteamManifestService { - private readonly IServiceProvider _services; + private readonly DeveLanCacheUIDbContext _dbContext; private readonly IHttpClientFactory _httpClientFactoryForManifestDownloads; private readonly ILogger _logger; private readonly string _manifestDirectory; - private static SemaphoreSlim _semaphoreSlim = new SemaphoreSlim(1, 1); - - public SteamManifestService(DeveLanCacheConfiguration deveLanCacheConfiguration, IServiceProvider services, IHttpClientFactory httpClientFactory, ILogger logger) + public SteamManifestService( + DeveLanCacheConfiguration deveLanCacheConfiguration, + DeveLanCacheUIDbContext dbContext, + IHttpClientFactory httpClientFactory, + ILogger logger) { - _services = services; + _dbContext = dbContext; _httpClientFactoryForManifestDownloads = httpClientFactory; _logger = logger; @@ -19,112 +21,91 @@ public SteamManifestService(DeveLanCacheConfiguration deveLanCacheConfiguration, _manifestDirectory = Path.Combine(deveLanCacheUIDataDirectory, "manifests"); } - public void TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) + public async Task TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) { - //This method could use some TPL Dataflow, I now use locking which should be okayish - if (!lanCacheLogEntryRaw.Request.Contains("/manifest/") || lanCacheLogEntryRaw.DownloadIdentifier == null) { _logger.LogError("Code bug: Trying to download manifest that isn't actually a manifest: {OriginalLogLine}", lanCacheLogEntryRaw.OriginalLogLine); return; } - _ = Task.Run(async () => - { - var fallbackPolicy = Policy - .Handle() - .FallbackAsync(async (ct) => - { - await Task.CompletedTask; - _logger.LogInformation("Manifest saving: All retries failed, skipping..."); - }); - - var retryPolicy = Policy - .Handle() - .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), - (exception, timeSpan, context) => - { - _logger.LogInformation("Manifest saving: An error occurred while trying to save changes: {Message}", exception.Message); - }); - - await fallbackPolicy.WrapAsync(retryPolicy).ExecuteAsync(async () => + var fallbackPolicy = Policy + .Handle() + .FallbackAsync(async (ct) => { - try - { - _semaphoreSlim.Wait(); - await using (var scope = _services.CreateAsyncScope()) - { - var theManifestUrlPart = lanCacheLogEntryRaw.Request.Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]; - - var everythingAfterManifest = theManifestUrlPart.Split("/manifest/", StringSplitOptions.RemoveEmptyEntries).Last(); - var manifestId = everythingAfterManifest.Split("/", StringSplitOptions.RemoveEmptyEntries).First(); - - //Replace invalid chars should dissalow reading any file you want :) - var manifestIdFileName = RemoveNonNumericCharacters(manifestId) + ".bin"; - var depotId = RemoveNonNumericCharacters(lanCacheLogEntryRaw.DownloadIdentifier!); - var depotIdAndManifestIdentifier = Path.Combine(depotId, manifestIdFileName); + await Task.CompletedTask; + _logger.LogInformation("Manifest saving: All retries failed, skipping..."); + }); + var retryPolicy = Policy + .Handle() + .WaitAndRetryAsync(3, retryAttempt => TimeSpan.FromSeconds(Math.Pow(2, retryAttempt)), + (exception, timeSpan, context) => + { + _logger.LogInformation("Manifest saving: An error occurred while trying to save changes: {Message}", exception.Message); + }); - using var dbContext = scope.ServiceProvider.GetRequiredService(); - var dbManifestFound = await dbContext.SteamManifests.FirstOrDefaultAsync(t => t.UniqueManifestIdentifier == depotIdAndManifestIdentifier); + await fallbackPolicy.WrapAsync(retryPolicy).ExecuteAsync(async () => + { + var theManifestUrlPart = lanCacheLogEntryRaw.Request.Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]; - if (dbManifestFound != null) - { - return; - } + var everythingAfterManifest = theManifestUrlPart.Split("/manifest/", StringSplitOptions.RemoveEmptyEntries).Last(); + var manifestId = everythingAfterManifest.Split("/", StringSplitOptions.RemoveEmptyEntries).First(); - var fullPath = Path.Combine(_manifestDirectory, depotIdAndManifestIdentifier); - Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + //Replace invalid chars should dissalow reading any file you want :) + var manifestIdFileName = RemoveNonNumericCharacters(manifestId) + ".bin"; + var depotId = RemoveNonNumericCharacters(lanCacheLogEntryRaw.DownloadIdentifier!); + var depotIdAndManifestIdentifier = Path.Combine(depotId, manifestIdFileName); + var dbManifestFound = await _dbContext.SteamManifests.FirstOrDefaultAsync(t => t.UniqueManifestIdentifier == depotIdAndManifestIdentifier); - var cachedUrl = $"http://lancache.steamcontent.com{theManifestUrlPart}"; - using var httpClient = _httpClientFactoryForManifestDownloads.CreateClient(); - httpClient.DefaultRequestHeaders.Add("Host", lanCacheLogEntryRaw.Host); - httpClient.DefaultRequestHeaders.Add("User-Agent", "Valve/Steam HTTP Client 1.0"); - httpClient.DefaultRequestHeaders.Referrer = LanCacheLogReaderHostedService.SkipLogLineReferrer; //Add this to ensure we don't process this line again - var manifestResponse = await httpClient.GetAsync(cachedUrl); + if (dbManifestFound != null) + { + return; + } + var fullPath = Path.Combine(_manifestDirectory, depotIdAndManifestIdentifier); + Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); + var cachedUrl = $"http://lancache.steamcontent.com{theManifestUrlPart}"; + using var httpClient = _httpClientFactoryForManifestDownloads.CreateClient(); + httpClient.DefaultRequestHeaders.Add("Host", lanCacheLogEntryRaw.Host); + httpClient.DefaultRequestHeaders.Add("User-Agent", "Valve/Steam HTTP Client 1.0"); + httpClient.DefaultRequestHeaders.Referrer = LanCacheLogReaderHostedService.SkipLogLineReferrer; //Add this to ensure we don't process this line again + var manifestResponse = await httpClient.GetAsync(cachedUrl); + if (!manifestResponse.IsSuccessStatusCode) + { + _logger.LogWarning("Warning: Tried to obtain manifest for: {DownloadIdentifier} but status code was: {StatusCode}", lanCacheLogEntryRaw.DownloadIdentifier, manifestResponse.StatusCode); + return; + } + var manifestBytes = await manifestResponse.Content.ReadAsByteArrayAsync(); - if (!manifestResponse.IsSuccessStatusCode) - { - _logger.LogWarning("Warning: Tried to obtain manifest for: {DownloadIdentifier} but status code was: {StatusCode}", lanCacheLogEntryRaw.DownloadIdentifier, manifestResponse.StatusCode); - return; - } - var manifestBytes = await manifestResponse.Content.ReadAsByteArrayAsync(); + var dbManifest = ManifestBytesToDbSteamManifest(manifestBytes, depotIdAndManifestIdentifier); - var dbManifest = ManifestBytesToDbSteamManifest(manifestBytes, depotIdAndManifestIdentifier); + if (dbManifest == null) + { + _logger.LogWarning("Could not get manifest for depot: {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); + return; + } - if (dbManifest == null) - { - _logger.LogWarning("Could not get manifest for depot: {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); - return; - } + var dbValue = _dbContext.SteamManifests.FirstOrDefault(t => t.DepotId == dbManifest.DepotId && t.CreationTime == dbManifest.CreationTime); + if (dbValue != null) + { + _dbContext.Entry(dbValue).CurrentValues.SetValues(dbManifest); + _logger.LogInformation("Updated manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); + } + else + { + await _dbContext.SteamManifests.AddAsync(dbManifest); + _logger.LogInformation("Added manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); + } - var dbValue = dbContext.SteamManifests.FirstOrDefault(t => t.DepotId == dbManifest.DepotId && t.CreationTime == dbManifest.CreationTime); - if (dbValue != null) - { - dbContext.Entry(dbValue).CurrentValues.SetValues(dbManifest); - _logger.LogInformation("Updated manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); - } - else - { - await dbContext.SteamManifests.AddAsync(dbManifest); - _logger.LogInformation("Added manifest for {DownloadIdentifier}", lanCacheLogEntryRaw.DownloadIdentifier); - } + await File.WriteAllBytesAsync(fullPath, manifestBytes); - await File.WriteAllBytesAsync(fullPath, manifestBytes); - await dbContext.SaveChangesAsync(); - } - } - finally - { - _semaphoreSlim.Release(); - } - }); + // Save changes happens in the caller }); } From e72f3298c9a5903122238e685bd88a1738a62b5d Mon Sep 17 00:00:00 2001 From: Devedse Date: Tue, 6 May 2025 15:41:12 +0200 Subject: [PATCH 05/15] Add dependencies --- DeveLanCacheUI_Backend/Program.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DeveLanCacheUI_Backend/Program.cs b/DeveLanCacheUI_Backend/Program.cs index f318450..6173b8b 100644 --- a/DeveLanCacheUI_Backend/Program.cs +++ b/DeveLanCacheUI_Backend/Program.cs @@ -67,11 +67,13 @@ public static void Main(string[] args) builder.Services.AddHostedService(); builder.Services.AddHostedService(); + builder.Services.AddHostedService(); builder.Services.AddHttpClient(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); if (deveLanCacheConfiguration.Feature_DirectSteamIntegration) { From 362e58610257670f47bd75a6e7cf07c915e6031a Mon Sep 17 00:00:00 2001 From: Devedse Date: Tue, 6 May 2025 22:35:35 +0200 Subject: [PATCH 06/15] Fix build --- .../LanCacheLogReaderHostedServiceTests.cs | 24 +- .../Db/DeveLanCacheUIDbContext.cs | 2 +- ...50506203509_AsyncLogEntryTable.Designer.cs | 220 ++++++++++++++++++ .../20250506203509_AsyncLogEntryTable.cs | 52 +++++ .../DeveLanCacheUIDbContextModelSnapshot.cs | 101 +++++++- DeveLanCacheUI_Backend/Program.cs | 5 +- 6 files changed, 388 insertions(+), 16 deletions(-) create mode 100644 DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.Designer.cs create mode 100644 DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.cs diff --git a/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs b/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs index 9f8e870..a675a18 100644 --- a/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs +++ b/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs @@ -20,7 +20,7 @@ public void TestTenLinesEach200_LF() var line = new string('a', 200); // a line with 200 characters var content = string.Join("\n", Enumerable.Repeat(line, 11)); // 11 such lines var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -37,7 +37,7 @@ public void TestLineExactly1024_LF() // Arrange var line = new string('b', 1023); var stream = MockStream(line + "\n"); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -56,7 +56,7 @@ public void TestLine1023And1025_LF() var line2 = new string('d', 1025); var content = $"{line1}\n{line2}\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -75,7 +75,7 @@ public void TestTenLinesEach200_CRLF() var line = new string('a', 200); var content = string.Join("\r\n", Enumerable.Repeat(line, 11)); var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -92,7 +92,7 @@ public void TestLineExactly1024_CRLF() // Arrange var line = new string('b', 1022); var stream = MockStream(line + "\r\n"); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -109,7 +109,7 @@ public void TestLineExactly1025_CRLF() // Arrange var line = new string('b', 1023); var stream = MockStream(line + "\r\n"); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -126,7 +126,7 @@ public void TestLineExactly1026_CRLF() // Arrange var line = new string('b', 1024); var stream = MockStream(line + "\r\n"); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -145,7 +145,7 @@ public void TestLine1023And1025_CRLF() var line2 = new string('d', 1025); var content = $"{line1}\r\n{line2}\r\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!); + var sut = new LanCacheLogReaderHostedService(null!, null!, null!); var cts = new CancellationTokenSource(); // Act @@ -164,7 +164,7 @@ public void When_InitialTotalBytesReadIsSetToSkipFirstLine_ThenShouldOnlyReadLas var initialTotalBytesRead = 4; var content = "abc\ndef\nghi\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!) + var sut = new LanCacheLogReaderHostedService(null!, null!, null!) { TotalBytesRead = initialTotalBytesRead }; @@ -186,7 +186,7 @@ public void When_InitialTotalBytesReadIsGreaterThanStreamLength_PositionIsAdjust // Arrange var content = "line1\nline2\nline3\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!) + var sut = new LanCacheLogReaderHostedService(null!, null!, null!) { TotalBytesRead = 500 }; @@ -210,7 +210,7 @@ public void When_DoubleNewLine_AtExactBufferLength_Works() var stringOfLength1024 = new string('a', 1023); var content = $"{stringOfLength1024}\n\nline2\nline3\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!) + var sut = new LanCacheLogReaderHostedService(null!, null!, null!) { TotalBytesRead = 0 }; @@ -235,7 +235,7 @@ public void When_DoubleNewLine_AtExactBufferLength_CRLF_Works() var stringOfLength1024 = new string('a', 1023); var content = $"{stringOfLength1024}\n\r\nline2\nline3\n"; var stream = MockStream(content); - var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!) + var sut = new LanCacheLogReaderHostedService(null!, null!, null!) { TotalBytesRead = 0 }; diff --git a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs index 582de4a..83efe2c 100644 --- a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs +++ b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs @@ -28,7 +28,7 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) .HasKey(pc => new { pc.SteamDepotId, pc.SteamAppId }); modelBuilder.Entity() - .HasIndex(pc => pc.LanCacheLogEntryRaw); + .OwnsOne(pc => pc.LanCacheLogEntryRaw); } } } diff --git a/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.Designer.cs b/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.Designer.cs new file mode 100644 index 0000000..ca62360 --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.Designer.cs @@ -0,0 +1,220 @@ +// +using System; +using DeveLanCacheUI_Backend.Db; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + [DbContext(typeof(DeveLanCacheUIDbContext))] + [Migration("20250506203509_AsyncLogEntryTable")] + partial class AsyncLogEntryTable + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("AsyncLogEntryProcessingQueueItems"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CacheHitBytes") + .HasColumnType("INTEGER"); + + b.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CacheMissBytes") + .HasColumnType("INTEGER"); + + b.Property("ClientIp") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DownloadIdentifier") + .HasColumnType("INTEGER"); + + b.Property("DownloadIdentifierString") + .HasColumnType("TEXT"); + + b.Property("LastUpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CacheIdentifier"); + + b.HasIndex("ClientIp"); + + b.ToTable("DownloadEvents"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => + { + b.Property("Key") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => + { + b.Property("SteamDepotId") + .HasColumnType("INTEGER"); + + b.Property("SteamAppId") + .HasColumnType("INTEGER"); + + b.HasKey("SteamDepotId", "SteamAppId"); + + b.ToTable("SteamDepots"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => + { + b.Property("DepotId") + .HasColumnType("INTEGER"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.Property("UniqueManifestIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("DepotId", "CreationTime"); + + b.HasIndex("UniqueManifestIdentifier") + .IsUnique(); + + b.ToTable("SteamManifests"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => + { + b1.Property("DbAsyncLogEntryProcessingQueueItemId") + .HasColumnType("INTEGER"); + + b1.Property("BodyBytesSent") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("BodyBytesSentLong") + .HasColumnType("INTEGER"); + + b1.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("DateTime") + .HasColumnType("TEXT"); + + b1.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b1.Property("ForwardedFor") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("Host") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("HttpRange") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("OriginalLogLine") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("ParseException") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("Referer") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("RemoteAddress") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("RemoteUser") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("Request") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("Status") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("TimeLocal") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("UpstreamCacheStatus") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("UserAgent") + .IsRequired() + .HasColumnType("TEXT"); + + b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); + + b1.ToTable("AsyncLogEntryProcessingQueueItems"); + + b1.WithOwner() + .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); + }); + + b.Navigation("LanCacheLogEntryRaw") + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.cs b/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.cs new file mode 100644 index 0000000..e03a5ab --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.cs @@ -0,0 +1,52 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + /// + public partial class AsyncLogEntryTable : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "AsyncLogEntryProcessingQueueItems", + columns: table => new + { + Id = table.Column(type: "INTEGER", nullable: false) + .Annotation("Sqlite:Autoincrement", true), + LanCacheLogEntryRaw_CacheIdentifier = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_RemoteAddress = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_ForwardedFor = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_RemoteUser = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_TimeLocal = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_Request = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_Status = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_BodyBytesSent = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_Referer = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_UserAgent = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_UpstreamCacheStatus = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_Host = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_HttpRange = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_OriginalLogLine = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_ParseException = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_DateTime = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_BodyBytesSentLong = table.Column(type: "INTEGER", nullable: false), + LanCacheLogEntryRaw_DownloadIdentifier = table.Column(type: "TEXT", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_AsyncLogEntryProcessingQueueItems", x => x.Id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "AsyncLogEntryProcessingQueueItems"); + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs index eedec3e..d6be1a2 100644 --- a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs +++ b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs @@ -15,7 +15,18 @@ partial class DeveLanCacheUIDbContextModelSnapshot : ModelSnapshot protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "9.0.0"); + modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("AsyncLogEntryProcessingQueueItems"); + }); modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => { @@ -112,6 +123,94 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("SteamManifests"); }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => + { + b1.Property("DbAsyncLogEntryProcessingQueueItemId") + .HasColumnType("INTEGER"); + + b1.Property("BodyBytesSent") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("BodyBytesSentLong") + .HasColumnType("INTEGER"); + + b1.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("DateTime") + .HasColumnType("TEXT"); + + b1.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b1.Property("ForwardedFor") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("Host") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("HttpRange") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("OriginalLogLine") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("ParseException") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("Referer") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("RemoteAddress") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("RemoteUser") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("Request") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("Status") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("TimeLocal") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("UpstreamCacheStatus") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("UserAgent") + .IsRequired() + .HasColumnType("TEXT"); + + b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); + + b1.ToTable("AsyncLogEntryProcessingQueueItems"); + + b1.WithOwner() + .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); + }); + + b.Navigation("LanCacheLogEntryRaw") + .IsRequired(); + }); #pragma warning restore 612, 618 } } diff --git a/DeveLanCacheUI_Backend/Program.cs b/DeveLanCacheUI_Backend/Program.cs index 6173b8b..a189e9b 100644 --- a/DeveLanCacheUI_Backend/Program.cs +++ b/DeveLanCacheUI_Backend/Program.cs @@ -72,8 +72,9 @@ public static void Main(string[] args) builder.Services.AddHttpClient(); builder.Services.AddSingleton(); - builder.Services.AddSingleton(); - builder.Services.AddSingleton(); + + builder.Services.AddScoped(); + builder.Services.AddScoped(); if (deveLanCacheConfiguration.Feature_DirectSteamIntegration) { From 307a5642ac99e8acdf318470fa20fadc49aa9585 Mon Sep 17 00:00:00 2001 From: Devedse Date: Tue, 6 May 2025 23:20:34 +0200 Subject: [PATCH 07/15] wip --- .../Db/DeveLanCacheUIDbContext.cs | 30 ++- .../LogReading/Models/LanCacheLogEntryRaw.cs | 1 + ...0506211600_AsyncLogEntryTable2.Designer.cs | 207 ++++++++++++++ .../20250506211600_AsyncLogEntryTable2.cs | 252 ++++++++++++++++++ ...0506212023_AsyncLogEntryTable3.Designer.cs | 207 ++++++++++++++ .../20250506212023_AsyncLogEntryTable3.cs | 22 ++ .../DeveLanCacheUIDbContextModelSnapshot.cs | 13 - 7 files changed, 718 insertions(+), 14 deletions(-) create mode 100644 DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.Designer.cs create mode 100644 DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.cs create mode 100644 DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.Designer.cs create mode 100644 DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.cs diff --git a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs index 83efe2c..c7c7d5d 100644 --- a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs +++ b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs @@ -27,8 +27,36 @@ protected override void OnModelCreating(ModelBuilder modelBuilder) modelBuilder.Entity() .HasKey(pc => new { pc.SteamDepotId, pc.SteamAppId }); + // Configure the owned entity to make eligible properties optional modelBuilder.Entity() - .OwnsOne(pc => pc.LanCacheLogEntryRaw); + .OwnsOne(pc => pc.LanCacheLogEntryRaw, ownedBuilder => + { + // Define which properties should remain required regardless of type + var requiredProperties = new HashSet { "CacheIdentifier", "OriginalLogLine" }; + + // Process each property + foreach (var property in typeof(LanCacheLogEntryRaw).GetProperties()) + { + // Skip required properties + if (requiredProperties.Contains(property.Name)) + continue; + + var propertyType = property.PropertyType; + + // Check if the property type can be nullable in the database: + // 1. Reference types (string, classes, etc.) + // 2. Already nullable value types (int?, DateTime?, etc.) + bool canBeNullable = !propertyType.IsValueType || + (propertyType.IsGenericType && + propertyType.GetGenericTypeDefinition() == typeof(Nullable<>)); + + if (canBeNullable) + { + // Only configure properties that can be nullable + ownedBuilder.Property(propertyType, property.Name).IsRequired(false); + } + } + }); } } } diff --git a/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs b/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs index 5b006e9..c9d13da 100644 --- a/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs +++ b/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs @@ -32,6 +32,7 @@ public class LanCacheLogEntryRaw public string HttpRange { get; init; } public required string OriginalLogLine { get; init; } + public string ParseException { get; init; } public DateTime DateTime { get; private set; } diff --git a/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.Designer.cs b/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.Designer.cs new file mode 100644 index 0000000..8ef1a6b --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.Designer.cs @@ -0,0 +1,207 @@ +// +using System; +using DeveLanCacheUI_Backend.Db; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + [DbContext(typeof(DeveLanCacheUIDbContext))] + [Migration("20250506211600_AsyncLogEntryTable2")] + partial class AsyncLogEntryTable2 + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("AsyncLogEntryProcessingQueueItems"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CacheHitBytes") + .HasColumnType("INTEGER"); + + b.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CacheMissBytes") + .HasColumnType("INTEGER"); + + b.Property("ClientIp") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DownloadIdentifier") + .HasColumnType("INTEGER"); + + b.Property("DownloadIdentifierString") + .HasColumnType("TEXT"); + + b.Property("LastUpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CacheIdentifier"); + + b.HasIndex("ClientIp"); + + b.ToTable("DownloadEvents"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => + { + b.Property("Key") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => + { + b.Property("SteamDepotId") + .HasColumnType("INTEGER"); + + b.Property("SteamAppId") + .HasColumnType("INTEGER"); + + b.HasKey("SteamDepotId", "SteamAppId"); + + b.ToTable("SteamDepots"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => + { + b.Property("DepotId") + .HasColumnType("INTEGER"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.Property("UniqueManifestIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("DepotId", "CreationTime"); + + b.HasIndex("UniqueManifestIdentifier") + .IsUnique(); + + b.ToTable("SteamManifests"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => + { + b1.Property("DbAsyncLogEntryProcessingQueueItemId") + .HasColumnType("INTEGER"); + + b1.Property("BodyBytesSent") + .HasColumnType("TEXT"); + + b1.Property("BodyBytesSentLong") + .HasColumnType("INTEGER"); + + b1.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("DateTime") + .HasColumnType("TEXT"); + + b1.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b1.Property("ForwardedFor") + .HasColumnType("TEXT"); + + b1.Property("Host") + .HasColumnType("TEXT"); + + b1.Property("HttpRange") + .HasColumnType("TEXT"); + + b1.Property("OriginalLogLine") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("ParseException") + .HasColumnType("TEXT"); + + b1.Property("Referer") + .HasColumnType("TEXT"); + + b1.Property("RemoteAddress") + .HasColumnType("TEXT"); + + b1.Property("RemoteUser") + .HasColumnType("TEXT"); + + b1.Property("Request") + .HasColumnType("TEXT"); + + b1.Property("Status") + .HasColumnType("TEXT"); + + b1.Property("TimeLocal") + .HasColumnType("TEXT"); + + b1.Property("UpstreamCacheStatus") + .HasColumnType("TEXT"); + + b1.Property("UserAgent") + .HasColumnType("TEXT"); + + b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); + + b1.ToTable("AsyncLogEntryProcessingQueueItems"); + + b1.WithOwner() + .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); + }); + + b.Navigation("LanCacheLogEntryRaw") + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.cs b/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.cs new file mode 100644 index 0000000..5691df7 --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.cs @@ -0,0 +1,252 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + /// + public partial class AsyncLogEntryTable2 : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_UserAgent", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_UpstreamCacheStatus", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_TimeLocal", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_Status", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_Request", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_RemoteUser", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_RemoteAddress", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_Referer", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_ParseException", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_HttpRange", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_Host", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_ForwardedFor", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_BodyBytesSent", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: true, + oldClrType: typeof(string), + oldType: "TEXT"); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_UserAgent", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_UpstreamCacheStatus", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_TimeLocal", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_Status", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_Request", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_RemoteUser", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_RemoteAddress", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_Referer", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_ParseException", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_HttpRange", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_Host", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_ForwardedFor", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + + migrationBuilder.AlterColumn( + name: "LanCacheLogEntryRaw_BodyBytesSent", + table: "AsyncLogEntryProcessingQueueItems", + type: "TEXT", + nullable: false, + defaultValue: "", + oldClrType: typeof(string), + oldType: "TEXT", + oldNullable: true); + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.Designer.cs b/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.Designer.cs new file mode 100644 index 0000000..0f3fc75 --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.Designer.cs @@ -0,0 +1,207 @@ +// +using System; +using DeveLanCacheUI_Backend.Db; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + [DbContext(typeof(DeveLanCacheUIDbContext))] + [Migration("20250506212023_AsyncLogEntryTable3")] + partial class AsyncLogEntryTable3 + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("AsyncLogEntryProcessingQueueItems"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CacheHitBytes") + .HasColumnType("INTEGER"); + + b.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CacheMissBytes") + .HasColumnType("INTEGER"); + + b.Property("ClientIp") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DownloadIdentifier") + .HasColumnType("INTEGER"); + + b.Property("DownloadIdentifierString") + .HasColumnType("TEXT"); + + b.Property("LastUpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CacheIdentifier"); + + b.HasIndex("ClientIp"); + + b.ToTable("DownloadEvents"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => + { + b.Property("Key") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => + { + b.Property("SteamDepotId") + .HasColumnType("INTEGER"); + + b.Property("SteamAppId") + .HasColumnType("INTEGER"); + + b.HasKey("SteamDepotId", "SteamAppId"); + + b.ToTable("SteamDepots"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => + { + b.Property("DepotId") + .HasColumnType("INTEGER"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.Property("UniqueManifestIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("DepotId", "CreationTime"); + + b.HasIndex("UniqueManifestIdentifier") + .IsUnique(); + + b.ToTable("SteamManifests"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => + { + b1.Property("DbAsyncLogEntryProcessingQueueItemId") + .HasColumnType("INTEGER"); + + b1.Property("BodyBytesSent") + .HasColumnType("TEXT"); + + b1.Property("BodyBytesSentLong") + .HasColumnType("INTEGER"); + + b1.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("DateTime") + .HasColumnType("TEXT"); + + b1.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b1.Property("ForwardedFor") + .HasColumnType("TEXT"); + + b1.Property("Host") + .HasColumnType("TEXT"); + + b1.Property("HttpRange") + .HasColumnType("TEXT"); + + b1.Property("OriginalLogLine") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("ParseException") + .HasColumnType("TEXT"); + + b1.Property("Referer") + .HasColumnType("TEXT"); + + b1.Property("RemoteAddress") + .HasColumnType("TEXT"); + + b1.Property("RemoteUser") + .HasColumnType("TEXT"); + + b1.Property("Request") + .HasColumnType("TEXT"); + + b1.Property("Status") + .HasColumnType("TEXT"); + + b1.Property("TimeLocal") + .HasColumnType("TEXT"); + + b1.Property("UpstreamCacheStatus") + .HasColumnType("TEXT"); + + b1.Property("UserAgent") + .HasColumnType("TEXT"); + + b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); + + b1.ToTable("AsyncLogEntryProcessingQueueItems"); + + b1.WithOwner() + .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); + }); + + b.Navigation("LanCacheLogEntryRaw") + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.cs b/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.cs new file mode 100644 index 0000000..f0b01d9 --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.cs @@ -0,0 +1,22 @@ +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + /// + public partial class AsyncLogEntryTable3 : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs index d6be1a2..1e43f77 100644 --- a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs +++ b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs @@ -132,7 +132,6 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("INTEGER"); b1.Property("BodyBytesSent") - .IsRequired() .HasColumnType("TEXT"); b1.Property("BodyBytesSentLong") @@ -149,15 +148,12 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("TEXT"); b1.Property("ForwardedFor") - .IsRequired() .HasColumnType("TEXT"); b1.Property("Host") - .IsRequired() .HasColumnType("TEXT"); b1.Property("HttpRange") - .IsRequired() .HasColumnType("TEXT"); b1.Property("OriginalLogLine") @@ -165,39 +161,30 @@ protected override void BuildModel(ModelBuilder modelBuilder) .HasColumnType("TEXT"); b1.Property("ParseException") - .IsRequired() .HasColumnType("TEXT"); b1.Property("Referer") - .IsRequired() .HasColumnType("TEXT"); b1.Property("RemoteAddress") - .IsRequired() .HasColumnType("TEXT"); b1.Property("RemoteUser") - .IsRequired() .HasColumnType("TEXT"); b1.Property("Request") - .IsRequired() .HasColumnType("TEXT"); b1.Property("Status") - .IsRequired() .HasColumnType("TEXT"); b1.Property("TimeLocal") - .IsRequired() .HasColumnType("TEXT"); b1.Property("UpstreamCacheStatus") - .IsRequired() .HasColumnType("TEXT"); b1.Property("UserAgent") - .IsRequired() .HasColumnType("TEXT"); b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); From 281295525b5408eed799a3ea06c4a3cbb07fbc10 Mon Sep 17 00:00:00 2001 From: Devedse Date: Tue, 6 May 2025 23:22:29 +0200 Subject: [PATCH 08/15] Undo migrations --- ...50506203509_AsyncLogEntryTable.Designer.cs | 220 --------------- .../20250506203509_AsyncLogEntryTable.cs | 52 ---- ...0506211600_AsyncLogEntryTable2.Designer.cs | 207 -------------- .../20250506211600_AsyncLogEntryTable2.cs | 252 ------------------ ...0506212023_AsyncLogEntryTable3.Designer.cs | 207 -------------- .../20250506212023_AsyncLogEntryTable3.cs | 22 -- .../DeveLanCacheUIDbContextModelSnapshot.cs | 212 +++++---------- 7 files changed, 63 insertions(+), 1109 deletions(-) delete mode 100644 DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.Designer.cs delete mode 100644 DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.cs delete mode 100644 DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.Designer.cs delete mode 100644 DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.cs delete mode 100644 DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.Designer.cs delete mode 100644 DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.cs diff --git a/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.Designer.cs b/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.Designer.cs deleted file mode 100644 index ca62360..0000000 --- a/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.Designer.cs +++ /dev/null @@ -1,220 +0,0 @@ -// -using System; -using DeveLanCacheUI_Backend.Db; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; - -#nullable disable - -namespace DeveLanCacheUI_Backend.Migrations -{ - [DbContext(typeof(DeveLanCacheUIDbContext))] - [Migration("20250506203509_AsyncLogEntryTable")] - partial class AsyncLogEntryTable - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("INTEGER"); - - b.HasKey("Id"); - - b.ToTable("AsyncLogEntryProcessingQueueItems"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("INTEGER"); - - b.Property("CacheHitBytes") - .HasColumnType("INTEGER"); - - b.Property("CacheIdentifier") - .IsRequired() - .HasColumnType("TEXT"); - - b.Property("CacheMissBytes") - .HasColumnType("INTEGER"); - - b.Property("ClientIp") - .IsRequired() - .HasColumnType("TEXT"); - - b.Property("CreatedAt") - .HasColumnType("TEXT"); - - b.Property("DownloadIdentifier") - .HasColumnType("INTEGER"); - - b.Property("DownloadIdentifierString") - .HasColumnType("TEXT"); - - b.Property("LastUpdatedAt") - .HasColumnType("TEXT"); - - b.HasKey("Id"); - - b.HasIndex("CacheIdentifier"); - - b.HasIndex("ClientIp"); - - b.ToTable("DownloadEvents"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => - { - b.Property("Key") - .HasColumnType("TEXT"); - - b.Property("Value") - .HasColumnType("TEXT"); - - b.HasKey("Key"); - - b.ToTable("Settings"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => - { - b.Property("SteamDepotId") - .HasColumnType("INTEGER"); - - b.Property("SteamAppId") - .HasColumnType("INTEGER"); - - b.HasKey("SteamDepotId", "SteamAppId"); - - b.ToTable("SteamDepots"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => - { - b.Property("DepotId") - .HasColumnType("INTEGER"); - - b.Property("CreationTime") - .HasColumnType("TEXT"); - - b.Property("ManifestBytesSize") - .HasColumnType("INTEGER"); - - b.Property("TotalCompressedSize") - .HasColumnType("INTEGER"); - - b.Property("TotalUncompressedSize") - .HasColumnType("INTEGER"); - - b.Property("UniqueManifestIdentifier") - .IsRequired() - .HasColumnType("TEXT"); - - b.HasKey("DepotId", "CreationTime"); - - b.HasIndex("UniqueManifestIdentifier") - .IsUnique(); - - b.ToTable("SteamManifests"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => - { - b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => - { - b1.Property("DbAsyncLogEntryProcessingQueueItemId") - .HasColumnType("INTEGER"); - - b1.Property("BodyBytesSent") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("BodyBytesSentLong") - .HasColumnType("INTEGER"); - - b1.Property("CacheIdentifier") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("DateTime") - .HasColumnType("TEXT"); - - b1.Property("DownloadIdentifier") - .HasColumnType("TEXT"); - - b1.Property("ForwardedFor") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("Host") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("HttpRange") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("OriginalLogLine") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("ParseException") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("Referer") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("RemoteAddress") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("RemoteUser") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("Request") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("Status") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("TimeLocal") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("UpstreamCacheStatus") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("UserAgent") - .IsRequired() - .HasColumnType("TEXT"); - - b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); - - b1.ToTable("AsyncLogEntryProcessingQueueItems"); - - b1.WithOwner() - .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); - }); - - b.Navigation("LanCacheLogEntryRaw") - .IsRequired(); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.cs b/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.cs deleted file mode 100644 index e03a5ab..0000000 --- a/DeveLanCacheUI_Backend/Migrations/20250506203509_AsyncLogEntryTable.cs +++ /dev/null @@ -1,52 +0,0 @@ -using System; -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace DeveLanCacheUI_Backend.Migrations -{ - /// - public partial class AsyncLogEntryTable : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.CreateTable( - name: "AsyncLogEntryProcessingQueueItems", - columns: table => new - { - Id = table.Column(type: "INTEGER", nullable: false) - .Annotation("Sqlite:Autoincrement", true), - LanCacheLogEntryRaw_CacheIdentifier = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_RemoteAddress = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_ForwardedFor = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_RemoteUser = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_TimeLocal = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_Request = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_Status = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_BodyBytesSent = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_Referer = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_UserAgent = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_UpstreamCacheStatus = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_Host = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_HttpRange = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_OriginalLogLine = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_ParseException = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_DateTime = table.Column(type: "TEXT", nullable: false), - LanCacheLogEntryRaw_BodyBytesSentLong = table.Column(type: "INTEGER", nullable: false), - LanCacheLogEntryRaw_DownloadIdentifier = table.Column(type: "TEXT", nullable: true) - }, - constraints: table => - { - table.PrimaryKey("PK_AsyncLogEntryProcessingQueueItems", x => x.Id); - }); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.DropTable( - name: "AsyncLogEntryProcessingQueueItems"); - } - } -} diff --git a/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.Designer.cs b/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.Designer.cs deleted file mode 100644 index 8ef1a6b..0000000 --- a/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.Designer.cs +++ /dev/null @@ -1,207 +0,0 @@ -// -using System; -using DeveLanCacheUI_Backend.Db; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; - -#nullable disable - -namespace DeveLanCacheUI_Backend.Migrations -{ - [DbContext(typeof(DeveLanCacheUIDbContext))] - [Migration("20250506211600_AsyncLogEntryTable2")] - partial class AsyncLogEntryTable2 - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("INTEGER"); - - b.HasKey("Id"); - - b.ToTable("AsyncLogEntryProcessingQueueItems"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("INTEGER"); - - b.Property("CacheHitBytes") - .HasColumnType("INTEGER"); - - b.Property("CacheIdentifier") - .IsRequired() - .HasColumnType("TEXT"); - - b.Property("CacheMissBytes") - .HasColumnType("INTEGER"); - - b.Property("ClientIp") - .IsRequired() - .HasColumnType("TEXT"); - - b.Property("CreatedAt") - .HasColumnType("TEXT"); - - b.Property("DownloadIdentifier") - .HasColumnType("INTEGER"); - - b.Property("DownloadIdentifierString") - .HasColumnType("TEXT"); - - b.Property("LastUpdatedAt") - .HasColumnType("TEXT"); - - b.HasKey("Id"); - - b.HasIndex("CacheIdentifier"); - - b.HasIndex("ClientIp"); - - b.ToTable("DownloadEvents"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => - { - b.Property("Key") - .HasColumnType("TEXT"); - - b.Property("Value") - .HasColumnType("TEXT"); - - b.HasKey("Key"); - - b.ToTable("Settings"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => - { - b.Property("SteamDepotId") - .HasColumnType("INTEGER"); - - b.Property("SteamAppId") - .HasColumnType("INTEGER"); - - b.HasKey("SteamDepotId", "SteamAppId"); - - b.ToTable("SteamDepots"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => - { - b.Property("DepotId") - .HasColumnType("INTEGER"); - - b.Property("CreationTime") - .HasColumnType("TEXT"); - - b.Property("ManifestBytesSize") - .HasColumnType("INTEGER"); - - b.Property("TotalCompressedSize") - .HasColumnType("INTEGER"); - - b.Property("TotalUncompressedSize") - .HasColumnType("INTEGER"); - - b.Property("UniqueManifestIdentifier") - .IsRequired() - .HasColumnType("TEXT"); - - b.HasKey("DepotId", "CreationTime"); - - b.HasIndex("UniqueManifestIdentifier") - .IsUnique(); - - b.ToTable("SteamManifests"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => - { - b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => - { - b1.Property("DbAsyncLogEntryProcessingQueueItemId") - .HasColumnType("INTEGER"); - - b1.Property("BodyBytesSent") - .HasColumnType("TEXT"); - - b1.Property("BodyBytesSentLong") - .HasColumnType("INTEGER"); - - b1.Property("CacheIdentifier") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("DateTime") - .HasColumnType("TEXT"); - - b1.Property("DownloadIdentifier") - .HasColumnType("TEXT"); - - b1.Property("ForwardedFor") - .HasColumnType("TEXT"); - - b1.Property("Host") - .HasColumnType("TEXT"); - - b1.Property("HttpRange") - .HasColumnType("TEXT"); - - b1.Property("OriginalLogLine") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("ParseException") - .HasColumnType("TEXT"); - - b1.Property("Referer") - .HasColumnType("TEXT"); - - b1.Property("RemoteAddress") - .HasColumnType("TEXT"); - - b1.Property("RemoteUser") - .HasColumnType("TEXT"); - - b1.Property("Request") - .HasColumnType("TEXT"); - - b1.Property("Status") - .HasColumnType("TEXT"); - - b1.Property("TimeLocal") - .HasColumnType("TEXT"); - - b1.Property("UpstreamCacheStatus") - .HasColumnType("TEXT"); - - b1.Property("UserAgent") - .HasColumnType("TEXT"); - - b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); - - b1.ToTable("AsyncLogEntryProcessingQueueItems"); - - b1.WithOwner() - .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); - }); - - b.Navigation("LanCacheLogEntryRaw") - .IsRequired(); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.cs b/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.cs deleted file mode 100644 index 5691df7..0000000 --- a/DeveLanCacheUI_Backend/Migrations/20250506211600_AsyncLogEntryTable2.cs +++ /dev/null @@ -1,252 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace DeveLanCacheUI_Backend.Migrations -{ - /// - public partial class AsyncLogEntryTable2 : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_UserAgent", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_UpstreamCacheStatus", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_TimeLocal", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_Status", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_Request", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_RemoteUser", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_RemoteAddress", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_Referer", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_ParseException", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_HttpRange", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_Host", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_ForwardedFor", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_BodyBytesSent", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: true, - oldClrType: typeof(string), - oldType: "TEXT"); - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_UserAgent", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_UpstreamCacheStatus", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_TimeLocal", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_Status", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_Request", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_RemoteUser", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_RemoteAddress", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_Referer", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_ParseException", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_HttpRange", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_Host", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_ForwardedFor", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - - migrationBuilder.AlterColumn( - name: "LanCacheLogEntryRaw_BodyBytesSent", - table: "AsyncLogEntryProcessingQueueItems", - type: "TEXT", - nullable: false, - defaultValue: "", - oldClrType: typeof(string), - oldType: "TEXT", - oldNullable: true); - } - } -} diff --git a/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.Designer.cs b/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.Designer.cs deleted file mode 100644 index 0f3fc75..0000000 --- a/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.Designer.cs +++ /dev/null @@ -1,207 +0,0 @@ -// -using System; -using DeveLanCacheUI_Backend.Db; -using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Infrastructure; -using Microsoft.EntityFrameworkCore.Migrations; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; - -#nullable disable - -namespace DeveLanCacheUI_Backend.Migrations -{ - [DbContext(typeof(DeveLanCacheUIDbContext))] - [Migration("20250506212023_AsyncLogEntryTable3")] - partial class AsyncLogEntryTable3 - { - /// - protected override void BuildTargetModel(ModelBuilder modelBuilder) - { -#pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("INTEGER"); - - b.HasKey("Id"); - - b.ToTable("AsyncLogEntryProcessingQueueItems"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("INTEGER"); - - b.Property("CacheHitBytes") - .HasColumnType("INTEGER"); - - b.Property("CacheIdentifier") - .IsRequired() - .HasColumnType("TEXT"); - - b.Property("CacheMissBytes") - .HasColumnType("INTEGER"); - - b.Property("ClientIp") - .IsRequired() - .HasColumnType("TEXT"); - - b.Property("CreatedAt") - .HasColumnType("TEXT"); - - b.Property("DownloadIdentifier") - .HasColumnType("INTEGER"); - - b.Property("DownloadIdentifierString") - .HasColumnType("TEXT"); - - b.Property("LastUpdatedAt") - .HasColumnType("TEXT"); - - b.HasKey("Id"); - - b.HasIndex("CacheIdentifier"); - - b.HasIndex("ClientIp"); - - b.ToTable("DownloadEvents"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => - { - b.Property("Key") - .HasColumnType("TEXT"); - - b.Property("Value") - .HasColumnType("TEXT"); - - b.HasKey("Key"); - - b.ToTable("Settings"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => - { - b.Property("SteamDepotId") - .HasColumnType("INTEGER"); - - b.Property("SteamAppId") - .HasColumnType("INTEGER"); - - b.HasKey("SteamDepotId", "SteamAppId"); - - b.ToTable("SteamDepots"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => - { - b.Property("DepotId") - .HasColumnType("INTEGER"); - - b.Property("CreationTime") - .HasColumnType("TEXT"); - - b.Property("ManifestBytesSize") - .HasColumnType("INTEGER"); - - b.Property("TotalCompressedSize") - .HasColumnType("INTEGER"); - - b.Property("TotalUncompressedSize") - .HasColumnType("INTEGER"); - - b.Property("UniqueManifestIdentifier") - .IsRequired() - .HasColumnType("TEXT"); - - b.HasKey("DepotId", "CreationTime"); - - b.HasIndex("UniqueManifestIdentifier") - .IsUnique(); - - b.ToTable("SteamManifests"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => - { - b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => - { - b1.Property("DbAsyncLogEntryProcessingQueueItemId") - .HasColumnType("INTEGER"); - - b1.Property("BodyBytesSent") - .HasColumnType("TEXT"); - - b1.Property("BodyBytesSentLong") - .HasColumnType("INTEGER"); - - b1.Property("CacheIdentifier") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("DateTime") - .HasColumnType("TEXT"); - - b1.Property("DownloadIdentifier") - .HasColumnType("TEXT"); - - b1.Property("ForwardedFor") - .HasColumnType("TEXT"); - - b1.Property("Host") - .HasColumnType("TEXT"); - - b1.Property("HttpRange") - .HasColumnType("TEXT"); - - b1.Property("OriginalLogLine") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("ParseException") - .HasColumnType("TEXT"); - - b1.Property("Referer") - .HasColumnType("TEXT"); - - b1.Property("RemoteAddress") - .HasColumnType("TEXT"); - - b1.Property("RemoteUser") - .HasColumnType("TEXT"); - - b1.Property("Request") - .HasColumnType("TEXT"); - - b1.Property("Status") - .HasColumnType("TEXT"); - - b1.Property("TimeLocal") - .HasColumnType("TEXT"); - - b1.Property("UpstreamCacheStatus") - .HasColumnType("TEXT"); - - b1.Property("UserAgent") - .HasColumnType("TEXT"); - - b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); - - b1.ToTable("AsyncLogEntryProcessingQueueItems"); - - b1.WithOwner() - .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); - }); - - b.Navigation("LanCacheLogEntryRaw") - .IsRequired(); - }); -#pragma warning restore 612, 618 - } - } -} diff --git a/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.cs b/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.cs deleted file mode 100644 index f0b01d9..0000000 --- a/DeveLanCacheUI_Backend/Migrations/20250506212023_AsyncLogEntryTable3.cs +++ /dev/null @@ -1,22 +0,0 @@ -using Microsoft.EntityFrameworkCore.Migrations; - -#nullable disable - -namespace DeveLanCacheUI_Backend.Migrations -{ - /// - public partial class AsyncLogEntryTable3 : Migration - { - /// - protected override void Up(MigrationBuilder migrationBuilder) - { - - } - - /// - protected override void Down(MigrationBuilder migrationBuilder) - { - - } - } -} diff --git a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs index 1e43f77..1b3f9ba 100644 --- a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs +++ b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs @@ -15,189 +15,103 @@ partial class DeveLanCacheUIDbContextModelSnapshot : ModelSnapshot protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("INTEGER"); - - b.HasKey("Id"); - - b.ToTable("AsyncLogEntryProcessingQueueItems"); - }); + modelBuilder.HasAnnotation("ProductVersion", "9.0.0"); modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("INTEGER"); + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); - b.Property("CacheHitBytes") - .HasColumnType("INTEGER"); + b.Property("CacheHitBytes") + .HasColumnType("INTEGER"); - b.Property("CacheIdentifier") - .IsRequired() - .HasColumnType("TEXT"); + b.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); - b.Property("CacheMissBytes") - .HasColumnType("INTEGER"); + b.Property("CacheMissBytes") + .HasColumnType("INTEGER"); - b.Property("ClientIp") - .IsRequired() - .HasColumnType("TEXT"); + b.Property("ClientIp") + .IsRequired() + .HasColumnType("TEXT"); - b.Property("CreatedAt") - .HasColumnType("TEXT"); + b.Property("CreatedAt") + .HasColumnType("TEXT"); - b.Property("DownloadIdentifier") - .HasColumnType("INTEGER"); + b.Property("DownloadIdentifier") + .HasColumnType("INTEGER"); - b.Property("DownloadIdentifierString") - .HasColumnType("TEXT"); + b.Property("DownloadIdentifierString") + .HasColumnType("TEXT"); - b.Property("LastUpdatedAt") - .HasColumnType("TEXT"); + b.Property("LastUpdatedAt") + .HasColumnType("TEXT"); - b.HasKey("Id"); + b.HasKey("Id"); - b.HasIndex("CacheIdentifier"); + b.HasIndex("CacheIdentifier"); - b.HasIndex("ClientIp"); + b.HasIndex("ClientIp"); - b.ToTable("DownloadEvents"); - }); + b.ToTable("DownloadEvents"); + }); modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => - { - b.Property("Key") - .HasColumnType("TEXT"); + { + b.Property("Key") + .HasColumnType("TEXT"); - b.Property("Value") - .HasColumnType("TEXT"); + b.Property("Value") + .HasColumnType("TEXT"); - b.HasKey("Key"); + b.HasKey("Key"); - b.ToTable("Settings"); - }); + b.ToTable("Settings"); + }); modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => - { - b.Property("SteamDepotId") - .HasColumnType("INTEGER"); + { + b.Property("SteamDepotId") + .HasColumnType("INTEGER"); - b.Property("SteamAppId") - .HasColumnType("INTEGER"); + b.Property("SteamAppId") + .HasColumnType("INTEGER"); - b.HasKey("SteamDepotId", "SteamAppId"); + b.HasKey("SteamDepotId", "SteamAppId"); - b.ToTable("SteamDepots"); - }); + b.ToTable("SteamDepots"); + }); modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => - { - b.Property("DepotId") - .HasColumnType("INTEGER"); - - b.Property("CreationTime") - .HasColumnType("TEXT"); - - b.Property("ManifestBytesSize") - .HasColumnType("INTEGER"); - - b.Property("TotalCompressedSize") - .HasColumnType("INTEGER"); - - b.Property("TotalUncompressedSize") - .HasColumnType("INTEGER"); - - b.Property("UniqueManifestIdentifier") - .IsRequired() - .HasColumnType("TEXT"); - - b.HasKey("DepotId", "CreationTime"); - - b.HasIndex("UniqueManifestIdentifier") - .IsUnique(); - - b.ToTable("SteamManifests"); - }); - - modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => - { - b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => - { - b1.Property("DbAsyncLogEntryProcessingQueueItemId") - .HasColumnType("INTEGER"); - - b1.Property("BodyBytesSent") - .HasColumnType("TEXT"); - - b1.Property("BodyBytesSentLong") - .HasColumnType("INTEGER"); - - b1.Property("CacheIdentifier") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("DateTime") - .HasColumnType("TEXT"); - - b1.Property("DownloadIdentifier") - .HasColumnType("TEXT"); - - b1.Property("ForwardedFor") - .HasColumnType("TEXT"); - - b1.Property("Host") - .HasColumnType("TEXT"); - - b1.Property("HttpRange") - .HasColumnType("TEXT"); - - b1.Property("OriginalLogLine") - .IsRequired() - .HasColumnType("TEXT"); - - b1.Property("ParseException") - .HasColumnType("TEXT"); - - b1.Property("Referer") - .HasColumnType("TEXT"); - - b1.Property("RemoteAddress") - .HasColumnType("TEXT"); - - b1.Property("RemoteUser") - .HasColumnType("TEXT"); - - b1.Property("Request") - .HasColumnType("TEXT"); + { + b.Property("DepotId") + .HasColumnType("INTEGER"); - b1.Property("Status") - .HasColumnType("TEXT"); + b.Property("CreationTime") + .HasColumnType("TEXT"); - b1.Property("TimeLocal") - .HasColumnType("TEXT"); + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); - b1.Property("UpstreamCacheStatus") - .HasColumnType("TEXT"); + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); - b1.Property("UserAgent") - .HasColumnType("TEXT"); + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); - b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); + b.Property("UniqueManifestIdentifier") + .IsRequired() + .HasColumnType("TEXT"); - b1.ToTable("AsyncLogEntryProcessingQueueItems"); + b.HasKey("DepotId", "CreationTime"); - b1.WithOwner() - .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); - }); + b.HasIndex("UniqueManifestIdentifier") + .IsUnique(); - b.Navigation("LanCacheLogEntryRaw") - .IsRequired(); - }); + b.ToTable("SteamManifests"); + }); #pragma warning restore 612, 618 } } From 63b3b04e66b9ba80b6172414c76489b8810da8d6 Mon Sep 17 00:00:00 2001 From: Devedse Date: Tue, 6 May 2025 23:23:54 +0200 Subject: [PATCH 09/15] Do migration --- ...50506212325_AsyncLogEntryTable.Designer.cs | 207 +++++++++++++++++ .../20250506212325_AsyncLogEntryTable.cs | 52 +++++ .../DeveLanCacheUIDbContextModelSnapshot.cs | 212 ++++++++++++------ 3 files changed, 408 insertions(+), 63 deletions(-) create mode 100644 DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.Designer.cs create mode 100644 DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.cs diff --git a/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.Designer.cs b/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.Designer.cs new file mode 100644 index 0000000..5204d20 --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.Designer.cs @@ -0,0 +1,207 @@ +// +using System; +using DeveLanCacheUI_Backend.Db; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + [DbContext(typeof(DeveLanCacheUIDbContext))] + [Migration("20250506212325_AsyncLogEntryTable")] + partial class AsyncLogEntryTable + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("AsyncLogEntryProcessingQueueItems"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CacheHitBytes") + .HasColumnType("INTEGER"); + + b.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CacheMissBytes") + .HasColumnType("INTEGER"); + + b.Property("ClientIp") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DownloadIdentifier") + .HasColumnType("INTEGER"); + + b.Property("DownloadIdentifierString") + .HasColumnType("TEXT"); + + b.Property("LastUpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CacheIdentifier"); + + b.HasIndex("ClientIp"); + + b.ToTable("DownloadEvents"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => + { + b.Property("Key") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => + { + b.Property("SteamDepotId") + .HasColumnType("INTEGER"); + + b.Property("SteamAppId") + .HasColumnType("INTEGER"); + + b.HasKey("SteamDepotId", "SteamAppId"); + + b.ToTable("SteamDepots"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => + { + b.Property("DepotId") + .HasColumnType("INTEGER"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.Property("UniqueManifestIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("DepotId", "CreationTime"); + + b.HasIndex("UniqueManifestIdentifier") + .IsUnique(); + + b.ToTable("SteamManifests"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => + { + b1.Property("DbAsyncLogEntryProcessingQueueItemId") + .HasColumnType("INTEGER"); + + b1.Property("BodyBytesSent") + .HasColumnType("TEXT"); + + b1.Property("BodyBytesSentLong") + .HasColumnType("INTEGER"); + + b1.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("DateTime") + .HasColumnType("TEXT"); + + b1.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b1.Property("ForwardedFor") + .HasColumnType("TEXT"); + + b1.Property("Host") + .HasColumnType("TEXT"); + + b1.Property("HttpRange") + .HasColumnType("TEXT"); + + b1.Property("OriginalLogLine") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("ParseException") + .HasColumnType("TEXT"); + + b1.Property("Referer") + .HasColumnType("TEXT"); + + b1.Property("RemoteAddress") + .HasColumnType("TEXT"); + + b1.Property("RemoteUser") + .HasColumnType("TEXT"); + + b1.Property("Request") + .HasColumnType("TEXT"); + + b1.Property("Status") + .HasColumnType("TEXT"); + + b1.Property("TimeLocal") + .HasColumnType("TEXT"); + + b1.Property("UpstreamCacheStatus") + .HasColumnType("TEXT"); + + b1.Property("UserAgent") + .HasColumnType("TEXT"); + + b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); + + b1.ToTable("AsyncLogEntryProcessingQueueItems"); + + b1.WithOwner() + .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); + }); + + b.Navigation("LanCacheLogEntryRaw") + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.cs b/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.cs new file mode 100644 index 0000000..7e2535e --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250506212325_AsyncLogEntryTable.cs @@ -0,0 +1,52 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + /// + public partial class AsyncLogEntryTable : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "AsyncLogEntryProcessingQueueItems", + columns: table => new + { + Id = table.Column(type: "INTEGER", nullable: false) + .Annotation("Sqlite:Autoincrement", true), + LanCacheLogEntryRaw_CacheIdentifier = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_RemoteAddress = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_ForwardedFor = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_RemoteUser = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_TimeLocal = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_Request = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_Status = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_BodyBytesSent = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_Referer = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_UserAgent = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_UpstreamCacheStatus = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_Host = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_HttpRange = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_OriginalLogLine = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_ParseException = table.Column(type: "TEXT", nullable: true), + LanCacheLogEntryRaw_DateTime = table.Column(type: "TEXT", nullable: false), + LanCacheLogEntryRaw_BodyBytesSentLong = table.Column(type: "INTEGER", nullable: false), + LanCacheLogEntryRaw_DownloadIdentifier = table.Column(type: "TEXT", nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_AsyncLogEntryProcessingQueueItems", x => x.Id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "AsyncLogEntryProcessingQueueItems"); + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs index 1b3f9ba..1e43f77 100644 --- a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs +++ b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs @@ -15,103 +15,189 @@ partial class DeveLanCacheUIDbContextModelSnapshot : ModelSnapshot protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 - modelBuilder.HasAnnotation("ProductVersion", "9.0.0"); + modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("AsyncLogEntryProcessingQueueItems"); + }); modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => - { - b.Property("Id") - .ValueGeneratedOnAdd() - .HasColumnType("INTEGER"); + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); - b.Property("CacheHitBytes") - .HasColumnType("INTEGER"); + b.Property("CacheHitBytes") + .HasColumnType("INTEGER"); - b.Property("CacheIdentifier") - .IsRequired() - .HasColumnType("TEXT"); + b.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); - b.Property("CacheMissBytes") - .HasColumnType("INTEGER"); + b.Property("CacheMissBytes") + .HasColumnType("INTEGER"); - b.Property("ClientIp") - .IsRequired() - .HasColumnType("TEXT"); + b.Property("ClientIp") + .IsRequired() + .HasColumnType("TEXT"); - b.Property("CreatedAt") - .HasColumnType("TEXT"); + b.Property("CreatedAt") + .HasColumnType("TEXT"); - b.Property("DownloadIdentifier") - .HasColumnType("INTEGER"); + b.Property("DownloadIdentifier") + .HasColumnType("INTEGER"); - b.Property("DownloadIdentifierString") - .HasColumnType("TEXT"); + b.Property("DownloadIdentifierString") + .HasColumnType("TEXT"); - b.Property("LastUpdatedAt") - .HasColumnType("TEXT"); + b.Property("LastUpdatedAt") + .HasColumnType("TEXT"); - b.HasKey("Id"); + b.HasKey("Id"); - b.HasIndex("CacheIdentifier"); + b.HasIndex("CacheIdentifier"); - b.HasIndex("ClientIp"); + b.HasIndex("ClientIp"); - b.ToTable("DownloadEvents"); - }); + b.ToTable("DownloadEvents"); + }); modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => - { - b.Property("Key") - .HasColumnType("TEXT"); + { + b.Property("Key") + .HasColumnType("TEXT"); - b.Property("Value") - .HasColumnType("TEXT"); + b.Property("Value") + .HasColumnType("TEXT"); - b.HasKey("Key"); + b.HasKey("Key"); - b.ToTable("Settings"); - }); + b.ToTable("Settings"); + }); modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => - { - b.Property("SteamDepotId") - .HasColumnType("INTEGER"); + { + b.Property("SteamDepotId") + .HasColumnType("INTEGER"); - b.Property("SteamAppId") - .HasColumnType("INTEGER"); + b.Property("SteamAppId") + .HasColumnType("INTEGER"); - b.HasKey("SteamDepotId", "SteamAppId"); + b.HasKey("SteamDepotId", "SteamAppId"); - b.ToTable("SteamDepots"); - }); + b.ToTable("SteamDepots"); + }); modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => - { - b.Property("DepotId") - .HasColumnType("INTEGER"); + { + b.Property("DepotId") + .HasColumnType("INTEGER"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.Property("UniqueManifestIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("DepotId", "CreationTime"); + + b.HasIndex("UniqueManifestIdentifier") + .IsUnique(); + + b.ToTable("SteamManifests"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => + { + b1.Property("DbAsyncLogEntryProcessingQueueItemId") + .HasColumnType("INTEGER"); + + b1.Property("BodyBytesSent") + .HasColumnType("TEXT"); + + b1.Property("BodyBytesSentLong") + .HasColumnType("INTEGER"); + + b1.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("DateTime") + .HasColumnType("TEXT"); + + b1.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b1.Property("ForwardedFor") + .HasColumnType("TEXT"); + + b1.Property("Host") + .HasColumnType("TEXT"); + + b1.Property("HttpRange") + .HasColumnType("TEXT"); + + b1.Property("OriginalLogLine") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("ParseException") + .HasColumnType("TEXT"); + + b1.Property("Referer") + .HasColumnType("TEXT"); + + b1.Property("RemoteAddress") + .HasColumnType("TEXT"); + + b1.Property("RemoteUser") + .HasColumnType("TEXT"); + + b1.Property("Request") + .HasColumnType("TEXT"); - b.Property("CreationTime") - .HasColumnType("TEXT"); + b1.Property("Status") + .HasColumnType("TEXT"); - b.Property("ManifestBytesSize") - .HasColumnType("INTEGER"); + b1.Property("TimeLocal") + .HasColumnType("TEXT"); - b.Property("TotalCompressedSize") - .HasColumnType("INTEGER"); + b1.Property("UpstreamCacheStatus") + .HasColumnType("TEXT"); - b.Property("TotalUncompressedSize") - .HasColumnType("INTEGER"); + b1.Property("UserAgent") + .HasColumnType("TEXT"); - b.Property("UniqueManifestIdentifier") - .IsRequired() - .HasColumnType("TEXT"); + b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); - b.HasKey("DepotId", "CreationTime"); + b1.ToTable("AsyncLogEntryProcessingQueueItems"); - b.HasIndex("UniqueManifestIdentifier") - .IsUnique(); + b1.WithOwner() + .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); + }); - b.ToTable("SteamManifests"); - }); + b.Navigation("LanCacheLogEntryRaw") + .IsRequired(); + }); #pragma warning restore 612, 618 } } From 36b246501170449e1f86cc8522985130a2343c52 Mon Sep 17 00:00:00 2001 From: Devedse Date: Tue, 6 May 2025 23:50:11 +0200 Subject: [PATCH 10/15] Added some more stuff and fixes --- ...tryProcessingQueueItemsProcessorHostedService.cs | 4 +++- .../Services/EpicManifestService.cs | 13 +++++++------ .../Services/SteamManifestService.cs | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/DeveLanCacheUI_Backend/LogReading/AsyncLogEntryProcessingQueueItemsProcessorHostedService.cs b/DeveLanCacheUI_Backend/LogReading/AsyncLogEntryProcessingQueueItemsProcessorHostedService.cs index d62ae52..fbb6547 100644 --- a/DeveLanCacheUI_Backend/LogReading/AsyncLogEntryProcessingQueueItemsProcessorHostedService.cs +++ b/DeveLanCacheUI_Backend/LogReading/AsyncLogEntryProcessingQueueItemsProcessorHostedService.cs @@ -30,7 +30,9 @@ private async Task GoRun(CancellationToken stoppingToken) { using var dbContext = scope.ServiceProvider.GetRequiredService(); - var items = await dbContext.AsyncLogEntryProcessingQueueItems.OrderBy(t => t.Id).ToListAsync(stoppingToken); + // Take 1 because I think it's nicer. We could take more but then we need to do a SaveChanges inside the manifest services. + // Because if you don't you're not getting the latest data. + var items = await dbContext.AsyncLogEntryProcessingQueueItems.OrderBy(t => t.Id).Take(1).ToListAsync(stoppingToken); if (items.Count == 0) { await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); diff --git a/DeveLanCacheUI_Backend/Services/EpicManifestService.cs b/DeveLanCacheUI_Backend/Services/EpicManifestService.cs index 58f56db..a9c4de2 100644 --- a/DeveLanCacheUI_Backend/Services/EpicManifestService.cs +++ b/DeveLanCacheUI_Backend/Services/EpicManifestService.cs @@ -24,12 +24,15 @@ public EpicManifestService( public async Task TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) { - if (!lanCacheLogEntryRaw.Request.Contains("/manifest/") || lanCacheLogEntryRaw.DownloadIdentifier == null) + if (lanCacheLogEntryRaw.CacheIdentifier != "epicgames" || !lanCacheLogEntryRaw.Request.Contains(".manifest?")) { _logger.LogError("Code bug: Trying to download manifest that isn't actually a manifest: {OriginalLogLine}", lanCacheLogEntryRaw.OriginalLogLine); return; } + var theManifestUrlPart = lanCacheLogEntryRaw.Request.Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]; + var fullManifestUrl = $"http://{lanCacheLogEntryRaw.Host}{theManifestUrlPart}"; + var fallbackPolicy = Policy .Handle() .FallbackAsync(async (ct) => @@ -48,14 +51,12 @@ public async Task TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) await fallbackPolicy.WrapAsync(retryPolicy).ExecuteAsync(async () => { - var theManifestUrlPart = lanCacheLogEntryRaw.Request.Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]; using var httpClient = _httpClientFactoryForManifestDownloads.CreateClient(); httpClient.DefaultRequestHeaders.Add("Host", lanCacheLogEntryRaw.Host); - httpClient.DefaultRequestHeaders.Add("User-Agent", "Valve/Steam HTTP Client 1.0"); + httpClient.DefaultRequestHeaders.Add("User-Agent", lanCacheLogEntryRaw.UserAgent); httpClient.DefaultRequestHeaders.Referrer = LanCacheLogReaderHostedService.SkipLogLineReferrer; //Add this to ensure we don't process this line again - var manifestResponse = await httpClient.GetAsync(theManifestUrlPart); - + var manifestResponse = await httpClient.GetAsync(fullManifestUrl); if (!manifestResponse.IsSuccessStatusCode) { @@ -66,7 +67,7 @@ await fallbackPolicy.WrapAsync(retryPolicy).ExecuteAsync(async () => var parsedManifest = EpicManifestParser.EpicManifestParser.Deserialize(manifestBytes); - _logger.LogInformation("Parsed epic manifest for Launch Exe: {Manifest}", parsedManifest.Meta.LaunchExe); + _logger.LogInformation("Parsed LogLine from: {LogLineDateTime}, Epic Manifest for AppId: '{AppId}' AppName: '{AppName}' Launch Exe: '{Manifest}'", lanCacheLogEntryRaw.DateTime, parsedManifest.Meta.AppID, parsedManifest.Meta.AppName, parsedManifest.Meta.LaunchExe); // Save changes happens in the caller }); diff --git a/DeveLanCacheUI_Backend/Services/SteamManifestService.cs b/DeveLanCacheUI_Backend/Services/SteamManifestService.cs index 4fbb6ce..eeb3372 100644 --- a/DeveLanCacheUI_Backend/Services/SteamManifestService.cs +++ b/DeveLanCacheUI_Backend/Services/SteamManifestService.cs @@ -23,7 +23,7 @@ public SteamManifestService( public async Task TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) { - if (!lanCacheLogEntryRaw.Request.Contains("/manifest/") || lanCacheLogEntryRaw.DownloadIdentifier == null) + if (lanCacheLogEntryRaw.CacheIdentifier != "steam" || !lanCacheLogEntryRaw.Request.Contains("/manifest/")) { _logger.LogError("Code bug: Trying to download manifest that isn't actually a manifest: {OriginalLogLine}", lanCacheLogEntryRaw.OriginalLogLine); return; From 7c43f15d864334aef56ad9da45b6ffbe9acd4ec5 Mon Sep 17 00:00:00 2001 From: Devedse Date: Tue, 6 May 2025 23:54:13 +0200 Subject: [PATCH 11/15] Fix test --- ...UI_Backend.EpicManifestParser.Tests.csproj | 26 +++++++------------ DeveLanCacheUI_Backend.Tests/UnitTest1.cs | 11 -------- 2 files changed, 9 insertions(+), 28 deletions(-) delete mode 100644 DeveLanCacheUI_Backend.Tests/UnitTest1.cs diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj index 2d8847b..af3792a 100644 --- a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj +++ b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/DeveLanCacheUI_Backend.EpicManifestParser.Tests.csproj @@ -2,34 +2,26 @@ net9.0 - latest enable enable - true - Exe - true - - true + + false - - - - + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + - - - - PreserveNewest diff --git a/DeveLanCacheUI_Backend.Tests/UnitTest1.cs b/DeveLanCacheUI_Backend.Tests/UnitTest1.cs deleted file mode 100644 index ea40382..0000000 --- a/DeveLanCacheUI_Backend.Tests/UnitTest1.cs +++ /dev/null @@ -1,11 +0,0 @@ -namespace DeveLanCacheUI_Backend.Tests -{ - [TestClass] - public class UnitTest1 - { - [TestMethod] - public void TestMethod1() - { - } - } -} \ No newline at end of file From b3b420f56623a4f63bbab9e15f9c62600757e267 Mon Sep 17 00:00:00 2001 From: Devedse Date: Tue, 6 May 2025 23:55:32 +0200 Subject: [PATCH 12/15] Fix tests 2 --- .../EpicManifestParserTests.cs | 4 +++- .../MSTestSettings.cs | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) delete mode 100644 DeveLanCacheUI_Backend.EpicManifestParser.Tests/MSTestSettings.cs diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs index d4db29a..ba7cf76 100644 --- a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs +++ b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs @@ -1,4 +1,6 @@ -namespace DeveLanCacheUI_Backend.EpicManifestParser.Tests +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace DeveLanCacheUI_Backend.EpicManifestParser.Tests { [TestClass] public sealed class EpicManifestParserTests diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/MSTestSettings.cs b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/MSTestSettings.cs deleted file mode 100644 index aaf278c..0000000 --- a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/MSTestSettings.cs +++ /dev/null @@ -1 +0,0 @@ -[assembly: Parallelize(Scope = ExecutionScope.MethodLevel)] From 26bf1d3fda494f0085043c8088d09a6cada2a5d2 Mon Sep 17 00:00:00 2001 From: Devedse Date: Fri, 9 May 2025 17:37:58 +0200 Subject: [PATCH 13/15] WIP adding epic manifest downloads --- .../EpicManifestParserTests.cs | 2 + .../Controllers/DownloadEventsController.cs | 46 +++++++++++++------ .../Controllers/Models/DownloadEvent.cs | 4 +- .../Controllers/Models/DownloadInfo.cs | 10 ++++ .../Db/DbModels/DbEpicManifest.cs | 16 +++++++ .../Db/DeveLanCacheUIDbContext.cs | 1 + .../LogReading/Models/LanCacheLogEntryRaw.cs | 16 ++++++- .../Services/EpicManifestService.cs | 20 +++++++- 8 files changed, 96 insertions(+), 19 deletions(-) create mode 100644 DeveLanCacheUI_Backend/Controllers/Models/DownloadInfo.cs create mode 100644 DeveLanCacheUI_Backend/Db/DbModels/DbEpicManifest.cs diff --git a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs index ba7cf76..2fd7f06 100644 --- a/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs +++ b/DeveLanCacheUI_Backend.EpicManifestParser.Tests/EpicManifestParserTests.cs @@ -16,6 +16,8 @@ public async Task DoesItWork() // Assert Assert.AreEqual("Super Space Club.exe", manifest.Meta.LaunchExe); + Assert.AreEqual("eccf14e21df84712a54d2b89b20d15f9", manifest.Meta.AppName); + Assert.AreEqual("9lEUV1B3tU-lv8teGLbqaQ", manifest.Meta.BuildId); } } } diff --git a/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs b/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs index 9be7d43..7e465d2 100644 --- a/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs +++ b/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs @@ -38,7 +38,7 @@ private async Task> GetFilteredBySkipAndCountInternal var excludedIps = _config.ExcludedClientIpsArray ?? Array.Empty(); IQueryable tmpResult = _dbContext.DownloadEvents; - + tmpResult = tmpResult.Where(t => !excludedIps.Contains(t.ClientIp)); if (filter != null) @@ -52,6 +52,8 @@ private async Task> GetFilteredBySkipAndCountInternal var query = from downloadEvent in pagedSubQuery join steamDepot in _dbContext.SteamDepots on downloadEvent.DownloadIdentifier equals steamDepot.SteamDepotId into steamDepotJoin from steamDepot in steamDepotJoin.DefaultIfEmpty() + join epicManifest in _dbContext.EpicManifests on downloadEvent.DownloadIdentifierString equals epicManifest.DownloadIdentifier into epicManifestJoin + from epicManifest in epicManifestJoin.DefaultIfEmpty() let steamManifest = (from sm in _dbContext.SteamManifests where sm.DepotId == downloadEvent.DownloadIdentifier orderby sm.CreationTime descending @@ -61,6 +63,7 @@ orderby downloadEvent.LastUpdatedAt descending { downloadEvent, steamDepot, + epicManifest, steamManifest }; @@ -72,8 +75,25 @@ orderby downloadEvent.LastUpdatedAt descending var mappedResult = groupedResult.Select(group => { - var firstItemInGroup = group.First(); + var firstItemInGroup = group + .FirstOrDefault(item => item.steamDepot != null) + ?? group.FirstOrDefault(item => item.epicManifest != null) + ?? group.First(); + + var downloadInfo = new DownloadInfo(); + if (group.Key.CacheIdentifier == "steam") + { + downloadInfo.Name = _steamAppObtainerService.GetSteamAppById(firstItemInGroup.steamDepot.SteamAppId)?.name; + downloadInfo.ClickUrl = $"https://steamdb.info/depot/{group.Key.DownloadIdentifierString}/"; + downloadInfo.ImageUrl = $"https://cdn.cloudflare.steamstatic.com/steam/apps/{firstItemInGroup.steamDepot.SteamAppId}/header.jpg"; + downloadInfo.TotalBytes = firstItemInGroup.steamManifest?.TotalCompressedSize ?? 0 + firstItemInGroup.steamManifest?.ManifestBytesSize ?? 0; + } + else if (group.Key.CacheIdentifier == "epicgames") + { + downloadInfo.Name = firstItemInGroup.epicManifest?.Name; + downloadInfo.TotalBytes = firstItemInGroup.epicManifest?.TotalCompressedSize ?? 0 + firstItemInGroup.epicManifest?.ManifestBytesSize ?? 0; + } var downloadEvent = new DownloadEvent { @@ -86,21 +106,17 @@ orderby downloadEvent.LastUpdatedAt descending LastUpdatedAt = group.Key.LastUpdatedAt, CacheHitBytes = group.Key.CacheHitBytes, CacheMissBytes = group.Key.CacheMissBytes, - TotalBytes = (firstItemInGroup.steamManifest?.TotalCompressedSize ?? 0) + (firstItemInGroup.steamManifest?.ManifestBytesSize ?? 0), - SteamDepot = group.Key.CacheIdentifier != "steam" || firstItemInGroup.steamDepot == null - ? null - : new SteamDepot - { - Id = firstItemInGroup.steamDepot.SteamDepotId, - SteamAppId = firstItemInGroup.steamDepot.SteamAppId - } + //TotalBytes = (firstItemInGroup.steamManifest?.TotalCompressedSize ?? 0) + (firstItemInGroup.steamManifest?.ManifestBytesSize ?? 0), + //SteamDepot = group.Key.CacheIdentifier != "steam" || firstItemInGroup.steamDepot == null + // ? null + // : new SteamDepot + // { + // Id = firstItemInGroup.steamDepot.SteamDepotId, + // SteamAppId = firstItemInGroup.steamDepot.SteamAppId + // } + DownloadInfo = downloadInfo }; - if (downloadEvent.CacheIdentifier == "steam" && downloadEvent.SteamDepot != null) - { - downloadEvent.SteamDepot.SteamApp = _steamAppObtainerService.GetSteamAppById(downloadEvent.SteamDepot.SteamAppId); - } - return downloadEvent; }).ToList(); diff --git a/DeveLanCacheUI_Backend/Controllers/Models/DownloadEvent.cs b/DeveLanCacheUI_Backend/Controllers/Models/DownloadEvent.cs index 5d0cb4a..8db605b 100644 --- a/DeveLanCacheUI_Backend/Controllers/Models/DownloadEvent.cs +++ b/DeveLanCacheUI_Backend/Controllers/Models/DownloadEvent.cs @@ -18,9 +18,9 @@ public class DownloadEvent public long CacheHitBytes { get; set; } public long CacheMissBytes { get; set; } - public ulong TotalBytes { get; set; } + //public ulong TotalBytes { get; set; } - public SteamDepot? SteamDepot { get; set; } + public DownloadInfo? DownloadInfo { get; set; } } } diff --git a/DeveLanCacheUI_Backend/Controllers/Models/DownloadInfo.cs b/DeveLanCacheUI_Backend/Controllers/Models/DownloadInfo.cs new file mode 100644 index 0000000..4a225ff --- /dev/null +++ b/DeveLanCacheUI_Backend/Controllers/Models/DownloadInfo.cs @@ -0,0 +1,10 @@ +namespace DeveLanCacheUI_Backend.Controllers.Models +{ + public class DownloadInfo + { + public string? Name { get; set; } + public string? ClickUrl { get; set; } + public string? ImageUrl { get; set; } + public ulong? TotalBytes { get; set; } + } +} diff --git a/DeveLanCacheUI_Backend/Db/DbModels/DbEpicManifest.cs b/DeveLanCacheUI_Backend/Db/DbModels/DbEpicManifest.cs new file mode 100644 index 0000000..acdcda1 --- /dev/null +++ b/DeveLanCacheUI_Backend/Db/DbModels/DbEpicManifest.cs @@ -0,0 +1,16 @@ +namespace DeveLanCacheUI_Backend.Db.DbModels +{ + [PrimaryKey(nameof(DownloadIdentifier), nameof(CreationTime))] + public class DbEpicManifest + { + public required string DownloadIdentifier { get; set; } + public required DateTime CreationTime { get; set; } + + public required ulong TotalCompressedSize { get; set; } + public required ulong TotalUncompressedSize { get; set; } + + public required ulong ManifestBytesSize { get; set; } + + public string Name { get; set; } + } +} diff --git a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs index c7c7d5d..64e03a9 100644 --- a/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs +++ b/DeveLanCacheUI_Backend/Db/DeveLanCacheUIDbContext.cs @@ -6,6 +6,7 @@ public class DeveLanCacheUIDbContext : DbContext public DbSet DownloadEvents => Set(); public DbSet Settings => Set(); public DbSet SteamManifests => Set(); + public DbSet EpicManifests => Set(); public DbSet AsyncLogEntryProcessingQueueItems => Set(); diff --git a/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs b/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs index c9d13da..e9f90bc 100644 --- a/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs +++ b/DeveLanCacheUI_Backend/LogReading/Models/LanCacheLogEntryRaw.cs @@ -70,7 +70,21 @@ public void CalculateFields() } else if (CacheIdentifier == "epicgames") { - DownloadIdentifier = "unknown"; + var urlPart = Request.Split(' ')[1]; + var splitted = urlPart.Split('/', StringSplitOptions.RemoveEmptyEntries); + + if (urlPart.StartsWith("/Builds/Org/") && (splitted.Length > 2)) + { + DownloadIdentifier = splitted[2]; + } + else if (urlPart.StartsWith("/ias/")) + { + DownloadIdentifier = splitted[1]; + } + else + { + DownloadIdentifier = "unknown"; + } } else if (CacheIdentifier == "riot") { diff --git a/DeveLanCacheUI_Backend/Services/EpicManifestService.cs b/DeveLanCacheUI_Backend/Services/EpicManifestService.cs index a9c4de2..1b02936 100644 --- a/DeveLanCacheUI_Backend/Services/EpicManifestService.cs +++ b/DeveLanCacheUI_Backend/Services/EpicManifestService.cs @@ -30,6 +30,13 @@ public async Task TryToDownloadManifest(LanCacheLogEntryRaw lanCacheLogEntryRaw) return; } + var firstItem = await _dbContext.EpicManifests.FirstOrDefaultAsync(t => t.DownloadIdentifier == lanCacheLogEntryRaw.DownloadIdentifier); + if (firstItem != null) + { + return; + } + + var theManifestUrlPart = lanCacheLogEntryRaw.Request.Split(' ', StringSplitOptions.RemoveEmptyEntries)[1]; var fullManifestUrl = $"http://{lanCacheLogEntryRaw.Host}{theManifestUrlPart}"; @@ -69,7 +76,18 @@ await fallbackPolicy.WrapAsync(retryPolicy).ExecuteAsync(async () => _logger.LogInformation("Parsed LogLine from: {LogLineDateTime}, Epic Manifest for AppId: '{AppId}' AppName: '{AppName}' Launch Exe: '{Manifest}'", lanCacheLogEntryRaw.DateTime, parsedManifest.Meta.AppID, parsedManifest.Meta.AppName, parsedManifest.Meta.LaunchExe); - // Save changes happens in the caller + var epicManifest = new DbEpicManifest() + { + Name = Path.GetFileNameWithoutExtension(parsedManifest.Meta.LaunchExe), + DownloadIdentifier = lanCacheLogEntryRaw.DownloadIdentifier, + CreationTime = DateTime.UtcNow, + ManifestBytesSize = (ulong)manifestBytes.Length, + TotalCompressedSize = (ulong)parsedManifest.TotalDownloadSize, + TotalUncompressedSize = (ulong)parsedManifest.TotalBuildSize + }; + + await _dbContext.EpicManifests.AddAsync(epicManifest); + await _dbContext.SaveChangesAsync(); }); } } From a2a0f5d2f1d291de3ff2c87639d05f3b20b84332 Mon Sep 17 00:00:00 2001 From: Devedse Date: Sun, 11 May 2025 13:07:59 +0200 Subject: [PATCH 14/15] WIP rewriting codebase a little --- .../Controllers/DownloadEventsController.cs | 17 +- .../Controllers/Models/DownloadInfo.cs | 11 +- .../Controllers/Models/SteamDepot.cs | 10 - .../Db/DbModels/DbDownloadEvent.cs | 1 + ...104218_EpicManifestsAndRewrite.Designer.cs | 233 ++++++++++++++++++ .../20250511104218_EpicManifestsAndRewrite.cs | 38 +++ .../DeveLanCacheUIDbContextModelSnapshot.cs | 26 ++ .../appsettings.Development.json | 4 +- 8 files changed, 319 insertions(+), 21 deletions(-) delete mode 100644 DeveLanCacheUI_Backend/Controllers/Models/SteamDepot.cs create mode 100644 DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.Designer.cs create mode 100644 DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.cs diff --git a/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs b/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs index 7e465d2..cfca075 100644 --- a/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs +++ b/DeveLanCacheUI_Backend/Controllers/DownloadEventsController.cs @@ -78,21 +78,24 @@ orderby downloadEvent.LastUpdatedAt descending var firstItemInGroup = group .FirstOrDefault(item => item.steamDepot != null) ?? group.FirstOrDefault(item => item.epicManifest != null) - ?? group.First(); + ?? group.FirstOrDefault(); var downloadInfo = new DownloadInfo(); if (group.Key.CacheIdentifier == "steam") { - downloadInfo.Name = _steamAppObtainerService.GetSteamAppById(firstItemInGroup.steamDepot.SteamAppId)?.name; - downloadInfo.ClickUrl = $"https://steamdb.info/depot/{group.Key.DownloadIdentifierString}/"; - downloadInfo.ImageUrl = $"https://cdn.cloudflare.steamstatic.com/steam/apps/{firstItemInGroup.steamDepot.SteamAppId}/header.jpg"; - downloadInfo.TotalBytes = firstItemInGroup.steamManifest?.TotalCompressedSize ?? 0 + firstItemInGroup.steamManifest?.ManifestBytesSize ?? 0; + downloadInfo.Name = _steamAppObtainerService.GetSteamAppById(firstItemInGroup?.steamDepot?.SteamAppId)?.name; + downloadInfo.AppUrl = $"https://steamdb.info/app/{firstItemInGroup?.steamDepot?.SteamAppId}/"; + downloadInfo.AppImageUrl = $"https://cdn.cloudflare.steamstatic.com/steam/apps/{firstItemInGroup?.steamDepot?.SteamAppId}/header.jpg"; + downloadInfo.DownloadIdentifier = group.Key.DownloadIdentifierString; + downloadInfo.DownloadIdentifierUrl = $"https://steamdb.info/depot/{group.Key.DownloadIdentifierString}"; + //downloadInfo.DownloadIdentifierImageUrl = $"https://shared.cloudflare.steamstatic.com/store_item_assets/steam/apps/{group.Key.DownloadIdentifierString}/header.jpg"; + downloadInfo.TotalBytes = firstItemInGroup?.steamManifest?.TotalCompressedSize ?? 0 + firstItemInGroup?.steamManifest?.ManifestBytesSize ?? 0; } else if (group.Key.CacheIdentifier == "epicgames") { - downloadInfo.Name = firstItemInGroup.epicManifest?.Name; - downloadInfo.TotalBytes = firstItemInGroup.epicManifest?.TotalCompressedSize ?? 0 + firstItemInGroup.epicManifest?.ManifestBytesSize ?? 0; + downloadInfo.Name = firstItemInGroup?.epicManifest?.Name; + downloadInfo.TotalBytes = firstItemInGroup?.epicManifest?.TotalCompressedSize ?? 0 + firstItemInGroup?.epicManifest?.ManifestBytesSize ?? 0; } var downloadEvent = new DownloadEvent diff --git a/DeveLanCacheUI_Backend/Controllers/Models/DownloadInfo.cs b/DeveLanCacheUI_Backend/Controllers/Models/DownloadInfo.cs index 4a225ff..4104f90 100644 --- a/DeveLanCacheUI_Backend/Controllers/Models/DownloadInfo.cs +++ b/DeveLanCacheUI_Backend/Controllers/Models/DownloadInfo.cs @@ -3,8 +3,15 @@ public class DownloadInfo { public string? Name { get; set; } - public string? ClickUrl { get; set; } - public string? ImageUrl { get; set; } + + public string? AppUrl { get; set; } + public string? AppImageUrl { get; set; } + + //E.g. the DepotId + public string? DownloadIdentifier { get; set; } + public string? DownloadIdentifierUrl { get; set; } + public string? DownloadIdentifierImageUrl { get; set; } + public ulong? TotalBytes { get; set; } } } diff --git a/DeveLanCacheUI_Backend/Controllers/Models/SteamDepot.cs b/DeveLanCacheUI_Backend/Controllers/Models/SteamDepot.cs deleted file mode 100644 index 48ea943..0000000 --- a/DeveLanCacheUI_Backend/Controllers/Models/SteamDepot.cs +++ /dev/null @@ -1,10 +0,0 @@ -namespace DeveLanCacheUI_Backend.Controllers.Models -{ - public class SteamDepot - { - public uint Id { get; set; } - - public App? SteamApp { get; set; } - public uint? SteamAppId { get; set; } - } -} diff --git a/DeveLanCacheUI_Backend/Db/DbModels/DbDownloadEvent.cs b/DeveLanCacheUI_Backend/Db/DbModels/DbDownloadEvent.cs index ced9831..a20ea83 100644 --- a/DeveLanCacheUI_Backend/Db/DbModels/DbDownloadEvent.cs +++ b/DeveLanCacheUI_Backend/Db/DbModels/DbDownloadEvent.cs @@ -8,6 +8,7 @@ public class DbDownloadEvent //steam/epicgames/wsus/epicgames public required string CacheIdentifier { get; set; } + //Ideally only use this for faster Joins, all other code should use the DownloadIdentifierString public uint? DownloadIdentifier { get; set; } public string? DownloadIdentifierString { get; set; } diff --git a/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.Designer.cs b/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.Designer.cs new file mode 100644 index 0000000..fba60d3 --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.Designer.cs @@ -0,0 +1,233 @@ +// +using System; +using DeveLanCacheUI_Backend.Db; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + [DbContext(typeof(DeveLanCacheUIDbContext))] + [Migration("20250511104218_EpicManifestsAndRewrite")] + partial class EpicManifestsAndRewrite + { + /// + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "9.0.4"); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.HasKey("Id"); + + b.ToTable("AsyncLogEntryProcessingQueueItems"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbDownloadEvent", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("CacheHitBytes") + .HasColumnType("INTEGER"); + + b.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CacheMissBytes") + .HasColumnType("INTEGER"); + + b.Property("ClientIp") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("CreatedAt") + .HasColumnType("TEXT"); + + b.Property("DownloadIdentifier") + .HasColumnType("INTEGER"); + + b.Property("DownloadIdentifierString") + .HasColumnType("TEXT"); + + b.Property("LastUpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.HasIndex("CacheIdentifier"); + + b.HasIndex("ClientIp"); + + b.ToTable("DownloadEvents"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbEpicManifest", b => + { + b.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.HasKey("DownloadIdentifier", "CreationTime"); + + b.ToTable("EpicManifests"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => + { + b.Property("Key") + .HasColumnType("TEXT"); + + b.Property("Value") + .HasColumnType("TEXT"); + + b.HasKey("Key"); + + b.ToTable("Settings"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamDepot", b => + { + b.Property("SteamDepotId") + .HasColumnType("INTEGER"); + + b.Property("SteamAppId") + .HasColumnType("INTEGER"); + + b.HasKey("SteamDepotId", "SteamAppId"); + + b.ToTable("SteamDepots"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSteamManifest", b => + { + b.Property("DepotId") + .HasColumnType("INTEGER"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.Property("UniqueManifestIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("DepotId", "CreationTime"); + + b.HasIndex("UniqueManifestIdentifier") + .IsUnique(); + + b.ToTable("SteamManifests"); + }); + + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbAsyncLogEntryProcessingQueueItem", b => + { + b.OwnsOne("DeveLanCacheUI_Backend.LogReading.Models.LanCacheLogEntryRaw", "LanCacheLogEntryRaw", b1 => + { + b1.Property("DbAsyncLogEntryProcessingQueueItemId") + .HasColumnType("INTEGER"); + + b1.Property("BodyBytesSent") + .HasColumnType("TEXT"); + + b1.Property("BodyBytesSentLong") + .HasColumnType("INTEGER"); + + b1.Property("CacheIdentifier") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("DateTime") + .HasColumnType("TEXT"); + + b1.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b1.Property("ForwardedFor") + .HasColumnType("TEXT"); + + b1.Property("Host") + .HasColumnType("TEXT"); + + b1.Property("HttpRange") + .HasColumnType("TEXT"); + + b1.Property("OriginalLogLine") + .IsRequired() + .HasColumnType("TEXT"); + + b1.Property("ParseException") + .HasColumnType("TEXT"); + + b1.Property("Referer") + .HasColumnType("TEXT"); + + b1.Property("RemoteAddress") + .HasColumnType("TEXT"); + + b1.Property("RemoteUser") + .HasColumnType("TEXT"); + + b1.Property("Request") + .HasColumnType("TEXT"); + + b1.Property("Status") + .HasColumnType("TEXT"); + + b1.Property("TimeLocal") + .HasColumnType("TEXT"); + + b1.Property("UpstreamCacheStatus") + .HasColumnType("TEXT"); + + b1.Property("UserAgent") + .HasColumnType("TEXT"); + + b1.HasKey("DbAsyncLogEntryProcessingQueueItemId"); + + b1.ToTable("AsyncLogEntryProcessingQueueItems"); + + b1.WithOwner() + .HasForeignKey("DbAsyncLogEntryProcessingQueueItemId"); + }); + + b.Navigation("LanCacheLogEntryRaw") + .IsRequired(); + }); +#pragma warning restore 612, 618 + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.cs b/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.cs new file mode 100644 index 0000000..ff72f20 --- /dev/null +++ b/DeveLanCacheUI_Backend/Migrations/20250511104218_EpicManifestsAndRewrite.cs @@ -0,0 +1,38 @@ +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +#nullable disable + +namespace DeveLanCacheUI_Backend.Migrations +{ + /// + public partial class EpicManifestsAndRewrite : Migration + { + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "EpicManifests", + columns: table => new + { + DownloadIdentifier = table.Column(type: "TEXT", nullable: false), + CreationTime = table.Column(type: "TEXT", nullable: false), + TotalCompressedSize = table.Column(type: "INTEGER", nullable: false), + TotalUncompressedSize = table.Column(type: "INTEGER", nullable: false), + ManifestBytesSize = table.Column(type: "INTEGER", nullable: false), + Name = table.Column(type: "TEXT", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_EpicManifests", x => new { x.DownloadIdentifier, x.CreationTime }); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "EpicManifests"); + } + } +} diff --git a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs index 1e43f77..11abef6 100644 --- a/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs +++ b/DeveLanCacheUI_Backend/Migrations/DeveLanCacheUIDbContextModelSnapshot.cs @@ -69,6 +69,32 @@ protected override void BuildModel(ModelBuilder modelBuilder) b.ToTable("DownloadEvents"); }); + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbEpicManifest", b => + { + b.Property("DownloadIdentifier") + .HasColumnType("TEXT"); + + b.Property("CreationTime") + .HasColumnType("TEXT"); + + b.Property("ManifestBytesSize") + .HasColumnType("INTEGER"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("TotalCompressedSize") + .HasColumnType("INTEGER"); + + b.Property("TotalUncompressedSize") + .HasColumnType("INTEGER"); + + b.HasKey("DownloadIdentifier", "CreationTime"); + + b.ToTable("EpicManifests"); + }); + modelBuilder.Entity("DeveLanCacheUI_Backend.Db.DbModels.DbSetting", b => { b.Property("Key") diff --git a/DeveLanCacheUI_Backend/appsettings.Development.json b/DeveLanCacheUI_Backend/appsettings.Development.json index 8739a29..f54bbca 100644 --- a/DeveLanCacheUI_Backend/appsettings.Development.json +++ b/DeveLanCacheUI_Backend/appsettings.Development.json @@ -1,11 +1,11 @@ { "Logging": { "LogLevel": { - "DeveLanCacheUI_Backend.Services": "Trace" + //"DeveLanCacheUI_Backend.Services": "Trace" } }, "DeveLanCacheUIDataDirectory": "DeveLanCacheUIData", - "LanCacheLogsDirectory": "\\\\10.88.20.1\\DockerComposers\\lancache\\logs", + "LanCacheLogsDirectory": "\\\\10.88.43.1\\root\\dockercomposers\\lancache\\logs", "Feature_DirectSteamIntegration": true, "Feature_SkipLinesBasedOnBytesRead": true, "ExcludedClientIps": "" From 4b36e142b38dee10f3178cbc7d229aee8b2a6d2d Mon Sep 17 00:00:00 2001 From: Devedse Date: Sun, 11 May 2025 22:49:53 +0200 Subject: [PATCH 15/15] Fix some warnings --- .../LanCacheLogReaderHostedServiceTests.cs | 20 +++--- .../LanCacheLogReaderHostedService.cs | 62 +++++++++---------- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs b/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs index a675a18..5d95dc5 100644 --- a/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs +++ b/DeveLanCacheUI_Backend.Tests/LogReading/LanCacheLogReaderHostedServiceTests.cs @@ -28,7 +28,7 @@ public void TestTenLinesEach200_LF() // Assert Assert.AreEqual(10 * 200 + 10, sut.TotalBytesRead); // 10 newlines added between 10 lines - data.ForEach(d => Assert.AreEqual(200, d.Length)); + data.ForEach(d => Assert.AreEqual(200, d!.Length)); } [TestMethod] @@ -45,7 +45,7 @@ public void TestLineExactly1024_LF() // Assert Assert.AreEqual(1024, sut.TotalBytesRead); - Assert.AreEqual(1023, data[0].Length); + Assert.AreEqual(1023, data[0]!.Length); } [TestMethod] @@ -64,8 +64,8 @@ public void TestLine1023And1025_LF() // Assert Assert.AreEqual(1023 + 1025 + 2, sut.TotalBytesRead); // +2 for the two '\n' sequences - Assert.AreEqual(1023, data[0].Length); - Assert.AreEqual(1025, data[1].Length); + Assert.AreEqual(1023, data[0]!.Length); + Assert.AreEqual(1025, data[1]!.Length); } [TestMethod] @@ -83,7 +83,7 @@ public void TestTenLinesEach200_CRLF() // Assert Assert.AreEqual(10 * 200 + 10 * 2, sut.TotalBytesRead); - data.ForEach(d => Assert.AreEqual(200, d.Length)); + data.ForEach(d => Assert.AreEqual(200, d!.Length)); } [TestMethod] @@ -100,7 +100,7 @@ public void TestLineExactly1024_CRLF() // Assert Assert.AreEqual(1024, sut.TotalBytesRead); - Assert.AreEqual(1022, data[0].Length); + Assert.AreEqual(1022, data[0]!.Length); } [TestMethod] @@ -117,7 +117,7 @@ public void TestLineExactly1025_CRLF() // Assert Assert.AreEqual(1025, sut.TotalBytesRead); - Assert.AreEqual(1023, data[0].Length); + Assert.AreEqual(1023, data[0]!.Length); } [TestMethod] @@ -134,7 +134,7 @@ public void TestLineExactly1026_CRLF() // Assert Assert.AreEqual(1026, sut.TotalBytesRead); - Assert.AreEqual(1024, data[0].Length); + Assert.AreEqual(1024, data[0]!.Length); } [TestMethod] @@ -153,8 +153,8 @@ public void TestLine1023And1025_CRLF() // Assert Assert.AreEqual(1023 + 1025 + 4, sut.TotalBytesRead); - Assert.AreEqual(1023, data[0].Length); - Assert.AreEqual(1025, data[1].Length); + Assert.AreEqual(1023, data[0]!.Length); + Assert.AreEqual(1025, data[1]!.Length); } [TestMethod] diff --git a/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs b/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs index ab34e64..6b121c4 100644 --- a/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs +++ b/DeveLanCacheUI_Backend/LogReading/LanCacheLogReaderHostedService.cs @@ -118,7 +118,7 @@ await retryPolicy.ExecuteAsync(async () => foreach (var lanCacheLogLine in filteredLogLines) { - if (lanCacheLogLine.CacheIdentifier == "steam" && ExcludedAppIds.Contains(lanCacheLogLine.DownloadIdentifier)) + if (lanCacheLogLine.CacheIdentifier == "steam" && lanCacheLogLine.DownloadIdentifier != null && ExcludedAppIds.Contains(lanCacheLogLine.DownloadIdentifier)) { continue; } @@ -257,39 +257,39 @@ public IEnumerable> Batch2(IEnumerable TailFrom(string file, CancellationToken stoppingToken) - { - using (var reader = File.OpenText(file)) - { - // go to end - if the next line is commented out, all the lines from the beginning is returned - // reader.BaseStream.Seek(0, SeekOrigin.End); - while (true) - { - stoppingToken.ThrowIfCancellationRequested(); - - string? line = reader.ReadLine(); - if (reader.BaseStream.Length < reader.BaseStream.Position) - { - Console.WriteLine($"Uhh: {reader.BaseStream.Length} < {reader.BaseStream.Position}"); - //reader.BaseStream.Seek(0, SeekOrigin.Begin); - - } - - if (line != null) - { - yield return line; - } - else - { - yield return null; - } - } - } - } + //static IEnumerable TailFrom(string file, CancellationToken stoppingToken) + //{ + // using (var reader = File.OpenText(file)) + // { + // // go to end - if the next line is commented out, all the lines from the beginning is returned + // // reader.BaseStream.Seek(0, SeekOrigin.End); + // while (true) + // { + // stoppingToken.ThrowIfCancellationRequested(); + + // string? line = reader.ReadLine(); + // if (reader.BaseStream.Length < reader.BaseStream.Position) + // { + // Console.WriteLine($"Uhh: {reader.BaseStream.Length} < {reader.BaseStream.Position}"); + // //reader.BaseStream.Seek(0, SeekOrigin.Begin); + + // } + + // if (line != null) + // { + // yield return line; + // } + // else + // { + // yield return null; + // } + // } + // } + //} public long TotalBytesRead { get; set; } - public IEnumerable TailFrom2(Stream inputStream, CancellationToken stoppingToken) + public IEnumerable TailFrom2(Stream inputStream, CancellationToken stoppingToken) { if (inputStream.Length >= TotalBytesRead) {