diff --git a/.gitignore b/.gitignore index d085e21..d281f58 100644 --- a/.gitignore +++ b/.gitignore @@ -359,3 +359,7 @@ local/ *.rej *.swp *.tmp + +# Agent rules +.cursor/* +.claude/* \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 1130ab2..49102a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -74,371 +74,27 @@ dependencies = [ ] [[package]] -name = "autocfg" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" - -[[package]] -name = "aws-config" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcdcf0d683fe9c23d32cf5b53c9918ea0a500375a9fb20109802552658e576c9" -dependencies = [ - "aws-credential-types", - "aws-http", - "aws-sdk-sso", - "aws-sdk-sts", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", - "bytes", - "fastrand 1.9.0", - "hex", - "http", - "hyper", - "ring 0.16.20", - "time", - "tokio", - "tower", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-credential-types" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fcdb2f7acbc076ff5ad05e7864bdb191ca70a6fd07668dc3a1a8bcd051de5ae" -dependencies = [ - "aws-smithy-async", - "aws-smithy-types", - "fastrand 1.9.0", - "tokio", - "tracing", - "zeroize", -] - -[[package]] -name = "aws-endpoint" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cce1c41a6cfaa726adee9ebb9a56fcd2bbfd8be49fd8a04c5e20fd968330b04" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "aws-types", - "http", - "regex", - "tracing", -] - -[[package]] -name = "aws-http" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aadbc44e7a8f3e71c8b374e03ecd972869eb91dd2bc89ed018954a52ba84bc44" -dependencies = [ - "aws-credential-types", - "aws-smithy-http", - "aws-smithy-types", - "aws-types", - "bytes", - "http", - "http-body", - "lazy_static", - "percent-encoding", - "pin-project-lite", - "tracing", -] - -[[package]] -name = "aws-sdk-s3" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fba197193cbb4bcb6aad8d99796b2291f36fa89562ded5d4501363055b0de89f" -dependencies = [ - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-sigv4", - "aws-smithy-async", - "aws-smithy-checksums", - "aws-smithy-client", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "http", - "http-body", - "once_cell", - "percent-encoding", - "regex", - "tokio-stream", - "tower", - "tracing", - "url", -] - -[[package]] -name = "aws-sdk-sso" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8b812340d86d4a766b2ca73f740dfd47a97c2dff0c06c8517a16d88241957e4" -dependencies = [ - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-types", - "aws-types", - "bytes", - "http", - "regex", - "tokio-stream", - "tower", - "tracing", -] - -[[package]] -name = "aws-sdk-sts" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "265fac131fbfc188e5c3d96652ea90ecc676a934e3174eaaee523c6cec040b3b" -dependencies = [ - "aws-credential-types", - "aws-endpoint", - "aws-http", - "aws-sig-auth", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "bytes", - "http", - "regex", - "tower", - "tracing", -] - -[[package]] -name = "aws-sig-auth" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b94acb10af0c879ecd5c7bdf51cda6679a0a4f4643ce630905a77673bfa3c61" -dependencies = [ - "aws-credential-types", - "aws-sigv4", - "aws-smithy-eventstream", - "aws-smithy-http", - "aws-types", - "http", - "tracing", -] - -[[package]] -name = "aws-sigv4" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d2ce6f507be68e968a33485ced670111d1cbad161ddbbab1e313c03d37d8f4c" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-http", - "bytes", - "form_urlencoded", - "hex", - "hmac", - "http", - "once_cell", - "percent-encoding", - "regex", - "sha2", - "time", - "tracing", -] - -[[package]] -name = "aws-smithy-async" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bda3996044c202d75b91afeb11a9afae9db9a721c6a7a427410018e286b880" -dependencies = [ - "futures-util", - "pin-project-lite", - "tokio", - "tokio-stream", -] - -[[package]] -name = "aws-smithy-checksums" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ed8b96d95402f3f6b8b57eb4e0e45ee365f78b1a924faf20ff6e97abf1eae6" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "crc32c", - "crc32fast", - "hex", - "http", - "http-body", - "md-5", - "pin-project-lite", - "sha1", - "sha2", - "tracing", -] - -[[package]] -name = "aws-smithy-client" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a86aa6e21e86c4252ad6a0e3e74da9617295d8d6e374d552be7d3059c41cedd" -dependencies = [ - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-http-tower", - "aws-smithy-types", - "bytes", - "fastrand 1.9.0", - "http", - "http-body", - "hyper", - "hyper-rustls", - "lazy_static", - "pin-project-lite", - "rustls", - "tokio", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-eventstream" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460c8da5110835e3d9a717c61f5556b20d03c32a1dec57f8fc559b360f733bb8" -dependencies = [ - "aws-smithy-types", - "bytes", - "crc32fast", -] - -[[package]] -name = "aws-smithy-http" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b3b693869133551f135e1f2c77cb0b8277d9e3e17feaf2213f735857c4f0d28" -dependencies = [ - "aws-smithy-eventstream", - "aws-smithy-types", - "bytes", - "bytes-utils", - "futures-core", - "http", - "http-body", - "hyper", - "once_cell", - "percent-encoding", - "pin-project-lite", - "pin-utils", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "aws-smithy-http-tower" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ae4f6c5798a247fac98a867698197d9ac22643596dc3777f0c76b91917616b9" -dependencies = [ - "aws-smithy-http", - "aws-smithy-types", - "bytes", - "http", - "http-body", - "pin-project-lite", - "tower", - "tracing", -] - -[[package]] -name = "aws-smithy-json" -version = "0.55.3" +name = "async-trait" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23f9f42fbfa96d095194a632fbac19f60077748eba536eb0b9fecc28659807f8" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ - "aws-smithy-types", -] - -[[package]] -name = "aws-smithy-query" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98819eb0b04020a1c791903533b638534ae6c12e2aceda3e6e6fba015608d51d" -dependencies = [ - "aws-smithy-types", - "urlencoding", -] - -[[package]] -name = "aws-smithy-types" -version = "0.55.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16a3d0bf4f324f4ef9793b86a1701d9700fbcdbd12a846da45eed104c634c6e8" -dependencies = [ - "base64-simd", - "itoa", - "num-integer", - "ryu", - "time", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "aws-smithy-xml" -version = "0.55.3" +name = "atomic-waker" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1b9d12875731bd07e767be7baad95700c3137b56730ec9ddeedb52a5e5ca63b" -dependencies = [ - "xmlparser", -] +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" [[package]] -name = "aws-types" -version = "0.55.3" +name = "autocfg" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dd209616cc8d7bfb82f87811a5c655dc97537f592689b18743bddf5dc5c4829" -dependencies = [ - "aws-credential-types", - "aws-smithy-async", - "aws-smithy-client", - "aws-smithy-http", - "aws-smithy-types", - "http", - "rustc_version", - "tracing", -] +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "backtrace" @@ -457,19 +113,9 @@ dependencies = [ [[package]] name = "base64" -version = "0.21.7" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" - -[[package]] -name = "base64-simd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" -dependencies = [ - "outref", - "vsimd", -] +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bitflags" @@ -498,16 +144,6 @@ version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" -[[package]] -name = "bytes-utils" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dafe3a8757b027e2be6e4e5601ed563c55989fcf1546e933c66c8eb3a058d35" -dependencies = [ - "bytes", - "either", -] - [[package]] name = "cc" version = "1.2.41" @@ -524,6 +160,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + [[package]] name = "chrono" version = "0.4.42" @@ -550,9 +192,9 @@ dependencies = [ [[package]] name = "core-foundation" -version = "0.9.4" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ "core-foundation-sys", "libc", @@ -564,33 +206,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "cpufeatures" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" -dependencies = [ - "libc", -] - -[[package]] -name = "crc32c" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" -dependencies = [ - "rustc_version", -] - -[[package]] -name = "crc32fast" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" -dependencies = [ - "cfg-if", -] - [[package]] name = "crypto-common" version = "0.1.6" @@ -601,15 +216,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "deranged" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a41953f86f8a05768a6cda24def994fd2f424b04ec5c719cf89989779f199071" -dependencies = [ - "powerfmt", -] - [[package]] name = "digest" version = "0.10.7" @@ -618,7 +224,6 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer", "crypto-common", - "subtle", ] [[package]] @@ -637,16 +242,16 @@ name = "drainage" version = "0.1.7" dependencies = [ "anyhow", - "aws-config", - "aws-sdk-s3", + "bytes", "chrono", "futures", "mockito", + "object_store", "pyo3", "serde", "serde_json", "tempfile", - "thiserror", + "thiserror 1.0.69", "tokio", "tokio-test", "url", @@ -674,15 +279,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "fastrand" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" -dependencies = [ - "instant", -] - [[package]] name = "fastrand" version = "2.3.0" @@ -816,8 +412,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", + "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", + "wasm-bindgen", ] [[package]] @@ -827,9 +425,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasi 0.14.7+wasi-0.2.4", + "wasm-bindgen", ] [[package]] @@ -849,7 +449,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.3.1", "indexmap", "slab", "tokio", @@ -865,30 +484,26 @@ checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" [[package]] name = "heck" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" - -[[package]] -name = "hex" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] -name = "hmac" -version = "0.12.1" +name = "http" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ - "digest", + "bytes", + "fnv", + "itoa", ] [[package]] name = "http" -version = "0.2.12" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565" dependencies = [ "bytes", "fnv", @@ -902,7 +517,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.3.1", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http 1.3.1", + "http-body 1.0.1", "pin-project-lite", ] @@ -918,6 +556,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" + [[package]] name = "hyper" version = "0.14.32" @@ -928,9 +572,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.27", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -942,19 +586,67 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2 0.4.12", + "http 1.3.1", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" -version = "0.23.2" +version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http", - "hyper", - "log", + "http 1.3.1", + "hyper 1.7.0", + "hyper-util", "rustls", "rustls-native-certs", + "rustls-pki-types", "tokio", "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "hyper 1.7.0", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2 0.6.1", + "tokio", + "tower-service", + "tracing", ] [[package]] @@ -1104,15 +796,6 @@ version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" -[[package]] -name = "instant" -version = "0.1.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" -dependencies = [ - "cfg-if", -] - [[package]] name = "io-uring" version = "0.7.10" @@ -1124,6 +807,31 @@ dependencies = [ "libc", ] +[[package]] +name = "ipnet" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" + +[[package]] +name = "iri-string" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.15" @@ -1179,6 +887,12 @@ version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + [[package]] name = "md-5" version = "0.10.6" @@ -1233,10 +947,10 @@ dependencies = [ "assert-json-diff", "colored", "futures", - "hyper", + "hyper 0.14.32", "lazy_static", "log", - "rand", + "rand 0.8.5", "regex", "serde_json", "serde_urlencoded", @@ -1244,21 +958,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "num-conv" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -1277,6 +976,43 @@ dependencies = [ "memchr", ] +[[package]] +name = "object_store" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c1be0c6c22ec0817cdc77d3842f721a17fd30ab6965001415b5402a74e6b740" +dependencies = [ + "async-trait", + "base64", + "bytes", + "chrono", + "form_urlencoded", + "futures", + "http 1.3.1", + "http-body-util", + "humantime", + "hyper 1.7.0", + "itertools", + "md-5", + "parking_lot", + "percent-encoding", + "quick-xml", + "rand 0.9.2", + "reqwest", + "ring", + "rustls-pemfile", + "serde", + "serde_json", + "serde_urlencoded", + "thiserror 2.0.17", + "tokio", + "tracing", + "url", + "walkdir", + "wasm-bindgen-futures", + "web-time", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -1289,12 +1025,6 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" -[[package]] -name = "outref" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" - [[package]] name = "parking_lot" version = "0.12.5" @@ -1324,26 +1054,6 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" -[[package]] -name = "pin-project" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "pin-project-lite" version = "0.2.16" @@ -1371,12 +1081,6 @@ dependencies = [ "zerovec", ] -[[package]] -name = "powerfmt" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1397,15 +1101,14 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.20.3" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +checksum = "fa8e48c12afdeb26aa4be4e5c49fb5e11c3efa0878db783a960eea2b9ac6dd19" dependencies = [ - "cfg-if", "indoc", "libc", "memoffset", - "parking_lot", + "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", @@ -1415,19 +1118,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.20.3" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +checksum = "bc1989dbf2b60852e0782c7487ebf0b4c7f43161ffe820849b56cf05f945cee1" dependencies = [ - "once_cell", "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.20.3" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +checksum = "c808286da7500385148930152e54fb6883452033085bf1f857d85d4e82ca905c" dependencies = [ "libc", "pyo3-build-config", @@ -1435,9 +1137,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.20.3" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +checksum = "83a0543c16be0d86cf0dbf2e2b636ece9fd38f20406bb43c255e0bc368095f92" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1447,9 +1149,9 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.20.3" +version = "0.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +checksum = "2a00da2ce064dcd582448ea24a5a26fa9527e0483103019b741ebcbe632dcd29" dependencies = [ "heck", "proc-macro2", @@ -1458,6 +1160,71 @@ dependencies = [ "syn", ] +[[package]] +name = "quick-xml" +version = "0.38.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "quinn" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2 0.6.1", + "thiserror 2.0.17", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" +dependencies = [ + "bytes", + "getrandom 0.3.3", + "lru-slab", + "rand 0.9.2", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror 2.0.17", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2 0.6.1", + "tracing", + "windows-sys 0.60.2", +] + [[package]] name = "quote" version = "1.0.41" @@ -1480,8 +1247,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" dependencies = [ "libc", - "rand_chacha", - "rand_core", + "rand_chacha 0.3.1", + "rand_core 0.6.4", +] + +[[package]] +name = "rand" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +dependencies = [ + "rand_chacha 0.9.0", + "rand_core 0.9.3", ] [[package]] @@ -1491,7 +1268,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core", + "rand_core 0.6.4", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.3", ] [[package]] @@ -1503,6 +1290,15 @@ dependencies = [ "getrandom 0.2.16", ] +[[package]] +name = "rand_core" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" +dependencies = [ + "getrandom 0.3.3", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1542,18 +1338,45 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] -name = "ring" -version = "0.16.20" +name = "reqwest" +version = "0.12.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ - "cc", - "libc", - "once_cell", - "spin", - "untrusted 0.7.1", + "base64", + "bytes", + "futures-core", + "futures-util", + "h2 0.4.12", + "http 1.3.1", + "http-body 1.0.1", + "http-body-util", + "hyper 1.7.0", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-native-certs", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", "web-sys", - "winapi", ] [[package]] @@ -1566,7 +1389,7 @@ dependencies = [ "cfg-if", "getrandom 0.2.16", "libc", - "untrusted 0.9.0", + "untrusted", "windows-sys 0.52.0", ] @@ -1577,13 +1400,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace" [[package]] -name = "rustc_version" -version = "0.4.1" +name = "rustc-hash" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustix" @@ -1600,35 +1420,58 @@ dependencies = [ [[package]] name = "rustls" -version = "0.20.9" +version = "0.23.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b80e3dec595989ea8510028f30c408a4630db12c9cbb8de34203b89d6577e99" +checksum = "751e04a496ca00bb97a5e043158d23d66b5aabf2e1d5aa2a0aaebb1aafe6f82c" dependencies = [ - "log", - "ring 0.16.20", - "sct", - "webpki", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" dependencies = [ "openssl-probe", - "rustls-pemfile", + "rustls-pki-types", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" -version = "1.0.4" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" dependencies = [ - "base64", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10b3f4191e8a80e6b43eebabfac91e5dcecebb27a71f04e820c47ec41d314bf" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", ] [[package]] @@ -1643,6 +1486,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "schannel" version = "0.1.28" @@ -1658,21 +1510,11 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring 0.17.14", - "untrusted 0.9.0", -] - [[package]] name = "security-framework" -version = "2.11.1" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ "bitflags", "core-foundation", @@ -1691,12 +1533,6 @@ dependencies = [ "libc", ] -[[package]] -name = "semver" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" - [[package]] name = "serde" version = "1.0.228" @@ -1752,28 +1588,6 @@ dependencies = [ "serde", ] -[[package]] -name = "sha1" -version = "0.10.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - -[[package]] -name = "sha2" -version = "0.10.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" -dependencies = [ - "cfg-if", - "cpufeatures", - "digest", -] - [[package]] name = "shlex" version = "1.3.0" @@ -1827,12 +1641,6 @@ dependencies = [ "windows-sys 0.60.2", ] -[[package]] -name = "spin" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" - [[package]] name = "stable_deref_trait" version = "1.2.1" @@ -1856,6 +1664,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1869,9 +1686,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.12.16" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" +checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" [[package]] name = "tempfile" @@ -1879,7 +1696,7 @@ version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ - "fastrand 2.3.0", + "fastrand", "getrandom 0.3.3", "once_cell", "rustix", @@ -1892,7 +1709,16 @@ version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" dependencies = [ - "thiserror-impl", + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" +dependencies = [ + "thiserror-impl 2.0.17", ] [[package]] @@ -1907,44 +1733,40 @@ dependencies = [ ] [[package]] -name = "time" -version = "0.3.44" +name = "thiserror-impl" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ - "deranged", - "num-conv", - "powerfmt", - "serde", - "time-core", - "time-macros", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "time-core" -version = "0.1.6" +name = "tinystr" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" +checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +dependencies = [ + "displaydoc", + "zerovec", +] [[package]] -name = "time-macros" -version = "0.2.24" +name = "tinyvec" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" dependencies = [ - "num-conv", - "time-core", + "tinyvec_macros", ] [[package]] -name = "tinystr" -version = "0.8.1" +name = "tinyvec_macros" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" -dependencies = [ - "displaydoc", - "zerovec", -] +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" @@ -1979,13 +1801,12 @@ dependencies = [ [[package]] name = "tokio-rustls" -version = "0.23.4" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls", "tokio", - "webpki", ] [[package]] @@ -2027,18 +1848,35 @@ dependencies = [ [[package]] name = "tower" -version = "0.4.13" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", - "pin-project", "pin-project-lite", + "sync_wrapper", "tokio", "tower-layer", "tower-service", - "tracing", +] + +[[package]] +name = "tower-http" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http 1.3.1", + "http-body 1.0.1", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", ] [[package]] @@ -2059,7 +1897,6 @@ version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ - "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2109,12 +1946,6 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" -[[package]] -name = "untrusted" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" - [[package]] name = "untrusted" version = "0.9.0" @@ -2133,12 +1964,6 @@ dependencies = [ "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf8_iter" version = "1.0.4" @@ -2152,10 +1977,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" [[package]] -name = "vsimd" -version = "0.8.0" +name = "walkdir" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] [[package]] name = "want" @@ -2217,6 +2046,19 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e038d41e478cc73bae0ff9b36c60cff1c98b8f38f8d7e8061e79ee63608ac5c" +dependencies = [ + "cfg-if", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.104" @@ -2250,46 +2092,46 @@ dependencies = [ ] [[package]] -name = "web-sys" -version = "0.3.81" +name = "wasm-streams" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" dependencies = [ + "futures-util", "js-sys", "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", ] [[package]] -name = "webpki" -version = "0.22.4" +name = "web-sys" +version = "0.3.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed63aea5ce73d0ff405984102c42de94fc55a6b75765d621c65262469b3c9b53" +checksum = "9367c417a924a74cae129e6a2ae3b47fabb1f8995595ab474029da749a8be120" dependencies = [ - "ring 0.17.14", - "untrusted 0.9.0", + "js-sys", + "wasm-bindgen", ] [[package]] -name = "winapi" -version = "0.3.9" +name = "web-time" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "js-sys", + "wasm-bindgen", ] [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" +name = "winapi-util" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] [[package]] name = "windows-core" @@ -2527,12 +2369,6 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" -[[package]] -name = "xmlparser" -version = "0.13.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66fee0b777b0f5ac1c69bb06d361268faafa61cd4682ae064a171c16c433e9e4" - [[package]] name = "yoke" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 0f07b29..fe6c5cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,10 +8,9 @@ name = "drainage" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.20", features = ["extension-module"] } +pyo3 = { version = "0.27", features = ["extension-module"] } tokio = { version = "1.0", features = ["full"] } -aws-sdk-s3 = "0.28" -aws-config = "0.55" +object_store = { version = "0.12", features = ["aws", "gcp"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" url = "2.4" @@ -19,9 +18,10 @@ chrono = { version = "0.4", features = ["serde"] } anyhow = "1.0" thiserror = "1.0" futures = "0.3" +bytes = "1.5" [dev-dependencies] tokio-test = "0.4" mockito = "0.32" tempfile = "3.8" -pyo3 = { version = "0.20", features = ["extension-module", "auto-initialize"] } +pyo3 = { version = "0.27", features = ["extension-module", "auto-initialize"] } diff --git a/README.md b/README.md index 5ac8880..7a866ba 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Rust + Python Lake House Health Analyzer Detect โ€ข Diagnose โ€ข Optimize โ€ข Flow -A high-performance Rust library with Python bindings for analyzing the health of remote S3-stored data lakes (Delta Lake and Apache Iceberg). +A high-performance Rust library with Python bindings for analyzing the health of cloud-stored data lakes (Delta Lake and Apache Iceberg). Currently supports Amazon S3 and Google Cloud Storage (GCS). Drainage helps you understand and optimize your data lake by identifying issues like unreferenced files, suboptimal partitioning, and inefficient file sizes. ## Features @@ -32,7 +32,9 @@ Drainage helps you understand and optimize your data lake by identifying issues - **๐Ÿ” Multi-Format Support**: - **Delta Lake tables** (including liquid clustering support) - **Apache Iceberg tables** (including clustering support) -- **โ˜๏ธ S3 Native**: Direct S3 integration for analyzing remote data lakes +- **โ˜๏ธ Multi-Cloud Support**: + - **AWS S3** + - **Google Cloud Storage (GCS)** - **๐Ÿ Python Interface**: Easy-to-use Python API powered by PyO3 - **๐Ÿงช Comprehensive Testing**: Full test suite with CI/CD across multiple platforms @@ -64,12 +66,18 @@ maturin develop --release ```python import drainage -# Analyze any table with automatic type detection +# Analyze S3 table with automatic type detection report = drainage.analyze_table( - s3_path="s3://my-bucket/my-table", + storage_path="s3://my-bucket/my-table", aws_region="us-west-2" ) +# or analyze GCS table +report = drainage.analyze_table( + storage_path="gs://my-bucket/my-table", + gcs_service_account_key="/path/to/service-account.json" +) + # Print a comprehensive health report drainage.print_health_report(report) @@ -84,14 +92,26 @@ print(f"Total Files: {report.metrics.total_files}") ```python import drainage -# Analyze a Delta Lake table +# Analyze a Delta Lake table on S3 report = drainage.analyze_delta_lake( - s3_path="s3://my-bucket/my-delta-table", + storage_path="s3://my-bucket/my-delta-table", aws_access_key_id="YOUR_ACCESS_KEY", # Optional if using IAM roles aws_secret_access_key="YOUR_SECRET_KEY", # Optional if using IAM roles aws_region="us-west-2" # Optional, defaults to us-east-1 ) +# Or analyze a Delta Lake table on GCS with service account +report = drainage.analyze_delta_lake( + storage_path="gs://my-bucket/my-delta-table", + gcs_service_account_key="/path/to/service-account.json" +) + +# Or use Application Default Credentials (ADC) for GCS +report = drainage.analyze_delta_lake( + storage_path="gs://my-bucket/my-delta-table" + # No credentials needed - will use ADC from environment +) + # View the health score (0.0 to 1.0) print(f"Health Score: {report.health_score}") @@ -111,12 +131,18 @@ for recommendation in report.metrics.recommendations: ```python import drainage -# Analyze an Apache Iceberg table +# Analyze an Apache Iceberg table on S3 report = drainage.analyze_iceberg( - s3_path="s3://my-bucket/my-iceberg-table", + storage_path="s3://my-bucket/my-iceberg-table", aws_region="us-west-2" ) +# Or analyze an Iceberg table on GCS +report = drainage.analyze_iceberg( + storage_path="gs://my-bucket/my-iceberg-table", + gcs_service_account_key="/path/to/service-account.json" +) + # View file size distribution dist = report.metrics.file_size_distribution print(f"Small files (<16MB): {dist.small_files}") @@ -147,12 +173,52 @@ for table in tables.collect(): print(f"โœ… Unity Catalog Delta table: {s3_path}") # Try analysis try: - report = drainage.analyze_table(s3_path=s3_path, aws_region="us-east-1") + report = drainage.analyze_table(storage_path=s3_path, aws_region="us-east-1") drainage.print_health_report(report) except Exception as e: print(f"โŒ Analysis failed: {e}") ``` +## Cloud Provider Authentication + +### AWS S3 + +**Using explicit credentials:** +```python +report = drainage.analyze_table( + storage_path="s3://my-bucket/my-table", + aws_access_key_id="YOUR_ACCESS_KEY", + aws_secret_access_key="YOUR_SECRET_KEY", + aws_region="us-west-2" +) +``` + +**Using IAM roles (recommended for EC2, Lambda, ECS, etc.):** +```python +report = drainage.analyze_table( + storage_path="s3://my-bucket/my-table", + aws_region="us-west-2" + # No credentials needed - will use IAM role attached to the instance +) +``` + +### Google Cloud Storage (GCS) + +**Using service account key file:** +```python +report = drainage.analyze_table( + storage_path="gs://my-bucket/my-table", + gcs_service_account_key="/path/to/service-account.json" +) +``` + +**Using Application Default Credentials (ADC):** +```python +report = drainage.analyze_table( + storage_path="gs://my-bucket/my-table" + # No credentials needed - will use ADC from environment +) +``` ## Health Metrics Explained diff --git a/examples/analyze_any_table.py b/examples/analyze_any_table.py index 775472c..46be899 100644 --- a/examples/analyze_any_table.py +++ b/examples/analyze_any_table.py @@ -49,9 +49,7 @@ def analyze_any_table( health_emoji = ( "๐ŸŸข" if report.health_score > 0.8 - else "๐ŸŸก" - if report.health_score > 0.6 - else "๐Ÿ”ด" + else "๐ŸŸก" if report.health_score > 0.6 else "๐Ÿ”ด" ) print(f"{health_emoji} Overall Health Score: {report.health_score:.1%}") print(f"๐Ÿ“… Analysis Timestamp: {report.analysis_timestamp}") diff --git a/examples/analyze_delta_table.py b/examples/analyze_delta_table.py index d2608e4..b2a130e 100644 --- a/examples/analyze_delta_table.py +++ b/examples/analyze_delta_table.py @@ -30,7 +30,7 @@ def analyze_delta_table(s3_path: str, aws_region: str = "us-west-2"): # Run the analysis report = drainage.analyze_delta_lake( s3_path=s3_path, - aws_region=aws_region + aws_region=aws_region, # aws_access_key_id=None, # Optional - uses default credentials # aws_secret_access_key=None, # Optional - uses default credentials ) @@ -44,9 +44,7 @@ def analyze_delta_table(s3_path: str, aws_region: str = "us-west-2"): health_emoji = ( "๐ŸŸข" if report.health_score > 0.8 - else "๐ŸŸก" - if report.health_score > 0.6 - else "๐Ÿ”ด" + else "๐ŸŸก" if report.health_score > 0.6 else "๐Ÿ”ด" ) print(f"{health_emoji} Overall Health Score: {report.health_score:.1%}") print(f"๐Ÿ“… Analysis Timestamp: {report.analysis_timestamp}\n") diff --git a/examples/analyze_iceberg_table.py b/examples/analyze_iceberg_table.py index fcad9e4..5178bb6 100644 --- a/examples/analyze_iceberg_table.py +++ b/examples/analyze_iceberg_table.py @@ -39,9 +39,7 @@ def analyze_iceberg_table(s3_path: str, aws_region: str = "us-west-2"): health_emoji = ( "๐ŸŸข" if report.health_score > 0.8 - else "๐ŸŸก" - if report.health_score > 0.6 - else "๐Ÿ”ด" + else "๐ŸŸก" if report.health_score > 0.6 else "๐Ÿ”ด" ) print(f"{health_emoji} Overall Health Score: {report.health_score:.1%}") print(f"๐Ÿ“… Analysis Timestamp: {report.analysis_timestamp}\n") diff --git a/examples/monitor_multiple_tables.py b/examples/monitor_multiple_tables.py index b03eefd..718f7b3 100644 --- a/examples/monitor_multiple_tables.py +++ b/examples/monitor_multiple_tables.py @@ -90,9 +90,7 @@ def monitor_tables(tables: List[Tuple[str, str]], aws_region: str = "us-west-2") health_emoji = ( "๐ŸŸข" if r["health_score"] > 0.8 - else "๐ŸŸก" - if r["health_score"] > 0.6 - else "๐Ÿ”ด" + else "๐ŸŸก" if r["health_score"] > 0.6 else "๐Ÿ”ด" ) path_short = r["path"][-35:] if len(r["path"]) > 35 else r["path"] print( diff --git a/pytest.ini b/pytest.ini index 98b10d4..587eab6 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,4 +1,4 @@ -[tool:pytest] +[pytest] testpaths = tests python_files = test_*.py python_classes = Test* diff --git a/src/delta_lake.rs b/src/delta_lake.rs index 3c4a0e5..c8a37b3 100644 --- a/src/delta_lake.rs +++ b/src/delta_lake.rs @@ -1,4 +1,4 @@ -use crate::s3_client::S3ClientWrapper; +use crate::storage_client::StorageClient; use crate::types::*; use anyhow::Result; use serde_json::Value; @@ -14,28 +14,28 @@ struct SchemaChange { } pub struct DeltaLakeAnalyzer { - s3_client: S3ClientWrapper, + storage_client: StorageClient, } impl DeltaLakeAnalyzer { - pub fn new(s3_client: S3ClientWrapper) -> Self { - Self { s3_client } + pub fn new(storage_client: StorageClient) -> Self { + Self { storage_client } } pub async fn analyze(&self) -> Result { let mut report = HealthReport::new( format!( - "s3://{}/{}", - self.s3_client.get_bucket(), - self.s3_client.get_prefix() + "{}/{}", + self.storage_client.get_bucket(), + self.storage_client.get_prefix() ), "delta".to_string(), ); // List all files in the Delta table directory let all_objects = self - .s3_client - .list_objects(self.s3_client.get_prefix()) + .storage_client + .list_objects(self.storage_client.get_prefix()) .await?; // Separate data files from metadata files @@ -55,7 +55,7 @@ impl DeltaLakeAnalyzer { // Find unreferenced files let referenced_set: HashSet = referenced_files.into_iter().collect(); for file in &data_files { - let file_path = format!("{}/{}", self.s3_client.get_prefix(), file.key); + let file_path = format!("{}/{}", self.storage_client.get_prefix(), file.key); if !referenced_set.contains(&file_path) { metrics.unreferenced_files.push(FileInfo { path: file_path, @@ -91,7 +91,7 @@ impl DeltaLakeAnalyzer { // Calculate additional health metrics metrics.calculate_data_skew(); - let metadata_files_owned: Vec = + let metadata_files_owned: Vec = metadata_files.iter().map(|f| (*f).clone()).collect(); metrics.calculate_metadata_health(&metadata_files_owned); metrics.calculate_snapshot_health(metadata_files.len()); // Simplified: use metadata file count as snapshot count @@ -126,10 +126,10 @@ impl DeltaLakeAnalyzer { fn categorize_files<'a>( &self, - objects: &'a [crate::s3_client::ObjectInfo], + objects: &'a [crate::storage_client::ObjectInfo], ) -> Result<( - Vec<&'a crate::s3_client::ObjectInfo>, - Vec<&'a crate::s3_client::ObjectInfo>, + Vec<&'a crate::storage_client::ObjectInfo>, + Vec<&'a crate::storage_client::ObjectInfo>, )> { let mut data_files = Vec::new(); let mut metadata_files = Vec::new(); @@ -147,12 +147,12 @@ impl DeltaLakeAnalyzer { async fn find_referenced_files( &self, - metadata_files: &[&crate::s3_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result> { let mut referenced_files = Vec::new(); for metadata_file in metadata_files { - let content = self.s3_client.get_object(&metadata_file.key).await?; + let content = self.storage_client.get_object(&metadata_file.key).await?; // Handle both single JSON objects and newline-delimited JSON (NDJSON) let content_str = String::from_utf8_lossy(&content); @@ -204,10 +204,10 @@ impl DeltaLakeAnalyzer { async fn find_clustering_info( &self, - metadata_files: &[&crate::s3_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result>> { for metadata_file in metadata_files { - let content = self.s3_client.get_object(&metadata_file.key).await?; + let content = self.storage_client.get_object(&metadata_file.key).await?; // Handle both single JSON objects and newline-delimited JSON (NDJSON) let content_str = String::from_utf8_lossy(&content); @@ -308,7 +308,7 @@ impl DeltaLakeAnalyzer { fn analyze_partitioning( &self, - data_files: &[&crate::s3_client::ObjectInfo], + data_files: &[&crate::storage_client::ObjectInfo], metrics: &mut HealthMetrics, ) -> Result<()> { let mut partition_map: HashMap = HashMap::new(); @@ -347,7 +347,7 @@ impl DeltaLakeAnalyzer { partition_info.file_count += 1; partition_info.total_size_bytes += file.size as u64; partition_info.files.push(FileInfo { - path: format!("{}/{}", self.s3_client.get_prefix(), file.key), + path: format!("{}/{}", self.storage_client.get_prefix(), file.key), size_bytes: file.size as u64, last_modified: file.last_modified.clone(), is_referenced: true, // We'll update this later @@ -370,7 +370,7 @@ impl DeltaLakeAnalyzer { fn analyze_clustering( &self, - data_files: &[&crate::s3_client::ObjectInfo], + data_files: &[&crate::storage_client::ObjectInfo], clustering_columns: &[String], metrics: &mut HealthMetrics, ) -> Result<()> { @@ -410,7 +410,7 @@ impl DeltaLakeAnalyzer { fn calculate_file_size_distribution( &self, - data_files: &[&crate::s3_client::ObjectInfo], + data_files: &[&crate::storage_client::ObjectInfo], metrics: &mut HealthMetrics, ) { for file in data_files { @@ -662,7 +662,7 @@ impl DeltaLakeAnalyzer { async fn analyze_schema_evolution( &self, - metadata_files: &[&crate::s3_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result> { let mut schema_changes = Vec::new(); let mut current_version = 0; @@ -679,7 +679,7 @@ impl DeltaLakeAnalyzer { }); for metadata_file in &sorted_files { - let content = self.s3_client.get_object(&metadata_file.key).await?; + let content = self.storage_client.get_object(&metadata_file.key).await?; let content_str = String::from_utf8_lossy(&content); for line in content_str.lines() { @@ -948,7 +948,7 @@ impl DeltaLakeAnalyzer { async fn analyze_deletion_vectors( &self, - metadata_files: &[&crate::s3_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result> { let mut deletion_vector_count = 0; let mut total_size = 0; @@ -956,7 +956,7 @@ impl DeltaLakeAnalyzer { let mut oldest_dv_age: f64 = 0.0; for metadata_file in metadata_files { - let content = self.s3_client.get_object(&metadata_file.key).await?; + let content = self.storage_client.get_object(&metadata_file.key).await?; let content_str = String::from_utf8_lossy(&content); for line in content_str.lines() { @@ -1094,7 +1094,7 @@ impl DeltaLakeAnalyzer { async fn analyze_time_travel( &self, - metadata_files: &[&crate::s3_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result> { let mut total_snapshots = 0; let mut total_historical_size = 0u64; @@ -1103,7 +1103,7 @@ impl DeltaLakeAnalyzer { // Analyze all metadata files to understand time travel storage for metadata_file in metadata_files { - let content = self.s3_client.get_object(&metadata_file.key).await?; + let content = self.storage_client.get_object(&metadata_file.key).await?; let content_str = String::from_utf8_lossy(&content); for line in content_str.lines() { @@ -1283,7 +1283,7 @@ impl DeltaLakeAnalyzer { async fn analyze_table_constraints( &self, - metadata_files: &[&crate::s3_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result> { let mut total_constraints = 0; let mut check_constraints = 0; @@ -1293,7 +1293,7 @@ impl DeltaLakeAnalyzer { // Analyze metadata files for constraint information for metadata_file in metadata_files { - let content = self.s3_client.get_object(&metadata_file.key).await?; + let content = self.storage_client.get_object(&metadata_file.key).await?; let content_str = String::from_utf8_lossy(&content); for line in content_str.lines() { @@ -1473,8 +1473,8 @@ impl DeltaLakeAnalyzer { async fn analyze_file_compaction( &self, - data_files: &[&crate::s3_client::ObjectInfo], - metadata_files: &[&crate::s3_client::ObjectInfo], + data_files: &[&crate::storage_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result> { let mut small_files_count = 0; let mut small_files_size = 0u64; @@ -1555,7 +1555,7 @@ impl DeltaLakeAnalyzer { fn calculate_recommended_target_size( &self, - data_files: &[&crate::s3_client::ObjectInfo], + data_files: &[&crate::storage_client::ObjectInfo], ) -> u64 { if data_files.is_empty() { return 128 * 1024 * 1024; // 128MB default @@ -1588,11 +1588,11 @@ impl DeltaLakeAnalyzer { async fn analyze_z_order_opportunity( &self, - metadata_files: &[&crate::s3_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result<(bool, Vec)> { // Look for clustering columns that could benefit from Z-ordering for metadata_file in metadata_files { - let content = self.s3_client.get_object(&metadata_file.key).await?; + let content = self.storage_client.get_object(&metadata_file.key).await?; let content_str = String::from_utf8_lossy(&content); for line in content_str.lines() { diff --git a/src/health_analyzer.rs b/src/health_analyzer.rs index bf75d70..7967686 100644 --- a/src/health_analyzer.rs +++ b/src/health_analyzer.rs @@ -1,12 +1,12 @@ use crate::delta_lake::DeltaLakeAnalyzer; use crate::iceberg::IcebergAnalyzer; -use crate::s3_client::S3ClientWrapper; +use crate::storage_client::StorageClient; use crate::types::HealthReport; use pyo3::prelude::*; #[pyclass] pub struct HealthAnalyzer { - s3_client: S3ClientWrapper, + storage_client: StorageClient, } #[pymethods] @@ -14,8 +14,8 @@ impl HealthAnalyzer { /// Get basic table information pub fn get_table_info(&self) -> PyResult<(String, String)> { Ok(( - self.s3_client.get_bucket().to_string(), - self.s3_client.get_prefix().to_string(), + self.storage_client.get_bucket().to_string(), + self.storage_client.get_prefix().to_string(), )) } } @@ -23,28 +23,33 @@ impl HealthAnalyzer { impl HealthAnalyzer { /// Create a new HealthAnalyzer asynchronously (internal use) pub async fn create_async( - s3_path: String, + storage_path: String, aws_access_key_id: Option, aws_secret_access_key: Option, aws_region: Option, + gcs_service_account_key: Option, ) -> PyResult { - let s3_client = S3ClientWrapper::new( - &s3_path, + let storage_client = StorageClient::new( + &storage_path, aws_access_key_id, aws_secret_access_key, aws_region, + gcs_service_account_key, ) .await .map_err(|e| { - pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to create S3 client: {}", e)) + pyo3::exceptions::PyRuntimeError::new_err(format!( + "Failed to create storage client: {}", + e + )) })?; - Ok(Self { s3_client }) + Ok(Self { storage_client }) } /// Analyze Delta Lake table health (internal use) pub async fn analyze_delta_lake(&self) -> PyResult { - let analyzer = DeltaLakeAnalyzer::new(self.s3_client.clone()); + let analyzer = DeltaLakeAnalyzer::new(self.storage_client.clone()); analyzer.analyze().await.map_err(|e| { pyo3::exceptions::PyRuntimeError::new_err(format!("Delta Lake analysis failed: {}", e)) }) @@ -52,16 +57,18 @@ impl HealthAnalyzer { /// Analyze Apache Iceberg table health (internal use) pub async fn analyze_iceberg(&self) -> PyResult { - let analyzer = IcebergAnalyzer::new(self.s3_client.clone()); + let analyzer = IcebergAnalyzer::new(self.storage_client.clone()); analyzer.analyze().await.map_err(|e| { pyo3::exceptions::PyRuntimeError::new_err(format!("Iceberg analysis failed: {}", e)) }) } /// List objects for table type detection (internal use) - pub async fn list_objects_for_detection(&self) -> PyResult> { - self.s3_client - .list_objects(self.s3_client.get_prefix()) + pub async fn list_objects_for_detection( + &self, + ) -> PyResult> { + self.storage_client + .list_objects(self.storage_client.get_prefix()) .await .map_err(|e| { pyo3::exceptions::PyRuntimeError::new_err(format!("Failed to list objects: {}", e)) @@ -69,25 +76,13 @@ impl HealthAnalyzer { } } -// We need to implement Clone for S3ClientWrapper to use it in the analyzer methods -impl Clone for S3ClientWrapper { - fn clone(&self) -> Self { - // This is a simplified clone - in practice, you might want to implement - // a more sophisticated cloning strategy or use Arc> for shared state - Self { - client: self.client.clone(), - bucket: self.bucket.clone(), - prefix: self.prefix.clone(), - } - } -} - #[cfg(test)] mod tests { + use crate::storage_client::*; #[test] fn test_health_analyzer_get_table_info() { - // This test would require a mock S3ClientWrapper + // This test would require a mock StorageClient // For now, we'll test the concept let bucket = "test-bucket".to_string(); let prefix = "test-prefix".to_string(); @@ -99,8 +94,8 @@ mod tests { } #[test] - fn test_health_analyzer_creation_parameters() { - let s3_path = "s3://test-bucket/test-table/"; + fn test_health_analyzer_creation_parameters_s3() { + let storage_path = "s3://test-bucket/test-table/"; let aws_access_key_id = Some("AKIAIOSFODNN7EXAMPLE".to_string()); let aws_secret_access_key = Some("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string()); let aws_region = Some("us-west-2".to_string()); @@ -109,44 +104,174 @@ mod tests { assert!(aws_access_key_id.is_some()); assert!(aws_secret_access_key.is_some()); assert!(aws_region.is_some()); + assert!(storage_path.starts_with("s3://")); + } + + #[test] + fn test_health_analyzer_creation_parameters_gcs() { + let storage_path = "gs://test-bucket/test-table/"; + let gcs_service_account_key = Some("/path/to/service-account.json".to_string()); + + assert!(gcs_service_account_key.is_some()); + assert!(storage_path.starts_with("gs://")); + } + + #[test] + fn test_health_analyzer_creation_without_credentials() { + let s3_path = "s3://test-bucket/test-table/"; + let aws_access_key_id: Option = None; + let aws_secret_access_key: Option = None; + let aws_region = Some("us-west-2".to_string()); + + // Test parameter validation for IAM role usage + assert!(aws_access_key_id.is_none()); + assert!(aws_secret_access_key.is_none()); + assert!(aws_region.is_some()); assert!(s3_path.starts_with("s3://")); } #[test] - fn test_health_analyzer_s3_path_validation() { + fn test_health_analyzer_creation_without_region() { + let s3_path = "s3://test-bucket/test-table/"; + let aws_access_key_id = Some("AKIAIOSFODNN7EXAMPLE".to_string()); + let aws_secret_access_key = Some("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string()); + let aws_region: Option = None; + + // Test parameter validation for default region + assert!(aws_access_key_id.is_some()); + assert!(aws_secret_access_key.is_some()); + assert!(aws_region.is_none()); + assert!(s3_path.starts_with("s3://")); + } + + #[test] + fn test_health_analyzer_gcs_creation_without_credentials() { + let gcs_path = "gs://test-bucket/test-table/"; + let gcs_service_account_key: Option = None; + + // Test parameter validation for ADC (Application Default Credentials) usage + assert!(gcs_service_account_key.is_none()); + assert!(gcs_path.starts_with("gs://")); + } + + #[test] + fn test_health_analyzer_gcs_creation_with_service_account() { + let gcs_path = "gs://test-bucket/test-table/"; + let gcs_service_account_key = Some("/path/to/service-account.json".to_string()); + + // Test parameter validation for service account key usage + assert!(gcs_service_account_key.is_some()); + assert!(gcs_path.starts_with("gs://")); + } + + #[test] + fn test_health_analyzer_storage_path_validation() { let valid_paths = vec![ "s3://bucket/table/", "s3://my-bucket/my-table/", "s3://bucket.with.dots/table/", "s3://bucket/path/to/table/", + "gs://bucket/table/", + "gs://my-bucket/my-table/", + "gs://bucket/path/to/table/", ]; for path in valid_paths { - assert!(path.starts_with("s3://"), "Invalid S3 path: {}", path); + let is_valid = path.starts_with("s3://") || path.starts_with("gs://"); + assert!(is_valid, "Invalid storage path: {}", path); assert!( path.contains("/"), - "S3 path should contain path separator: {}", + "Storage path should contain path separator: {}", path ); } } + #[test] + fn test_health_analyzer_storage_path_validation_invalid() { + let invalid_paths = vec![ + "https://bucket/table/", + "ftp://bucket/table/", + "not-a-url", + "", + "s3://", + "s3:///", + "gs://", + "gs:///", + ]; + + for path in invalid_paths { + if path.is_empty() { + continue; // Skip empty string test + } + let is_invalid = !path.starts_with("s3://") && !path.starts_with("gs://") + || path == "s3://" + || path == "s3:///" + || path == "gs://" + || path == "gs:///"; + assert!(is_invalid, "Should be invalid storage path: {}", path); + } + } + + #[test] + fn test_health_analyzer_aws_region_validation() { + let valid_regions = vec![ + "us-east-1", + "us-west-2", + "eu-west-1", + "ap-southeast-1", + "ca-central-1", + ]; + + for region in valid_regions { + assert!(!region.is_empty(), "Region should not be empty"); + assert!( + region.contains("-"), + "Region should contain dash: {}", + region + ); + } + } + + #[test] + fn test_health_analyzer_aws_credentials_validation() { + let valid_access_key = "AKIAIOSFODNN7EXAMPLE"; + let valid_secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + + assert!( + valid_access_key.starts_with("AKIA"), + "Access key should start with AKIA" + ); + assert!( + valid_secret_key.len() >= 20, + "Secret key should be at least 20 characters" + ); + assert!( + !valid_access_key.contains(" "), + "Access key should not contain spaces" + ); + assert!( + !valid_secret_key.contains(" "), + "Secret key should not contain spaces" + ); + } + #[test] fn test_health_analyzer_table_type_detection_delta() { let objects = vec![ - crate::s3_client::ObjectInfo { + ObjectInfo { key: "part-00000.parquet".to_string(), size: 1024, last_modified: None, etag: None, }, - crate::s3_client::ObjectInfo { + ObjectInfo { key: "_delta_log/00000000000000000000.json".to_string(), size: 2048, last_modified: None, etag: None, }, - crate::s3_client::ObjectInfo { + ObjectInfo { key: "_delta_log/00000000000000000001.json".to_string(), size: 1024, last_modified: None, @@ -167,19 +292,19 @@ mod tests { #[test] fn test_health_analyzer_table_type_detection_iceberg() { let objects = vec![ - crate::s3_client::ObjectInfo { + ObjectInfo { key: "data/00000-0-00000000000000000000-00000000000000000000.parquet".to_string(), size: 1024, last_modified: None, etag: None, }, - crate::s3_client::ObjectInfo { + ObjectInfo { key: "metadata/00000-00000000000000000000.metadata.json".to_string(), size: 2048, last_modified: None, etag: None, }, - crate::s3_client::ObjectInfo { + ObjectInfo { key: "metadata/snap-00000000000000000000-1-00000000000000000000.avro".to_string(), size: 1024, last_modified: None, @@ -200,19 +325,19 @@ mod tests { #[test] fn test_health_analyzer_table_type_detection_ambiguous() { let objects = vec![ - crate::s3_client::ObjectInfo { + ObjectInfo { key: "part-00000.parquet".to_string(), size: 1024, last_modified: None, etag: None, }, - crate::s3_client::ObjectInfo { + ObjectInfo { key: "_delta_log/00000000000000000000.json".to_string(), size: 2048, last_modified: None, etag: None, }, - crate::s3_client::ObjectInfo { + ObjectInfo { key: "metadata/00000-00000000000000000000.metadata.json".to_string(), size: 1024, last_modified: None, @@ -232,13 +357,41 @@ mod tests { } #[test] - fn test_health_analyzer_s3_client_wrapper_clone() { - // Test that S3ClientWrapper can be cloned + fn test_health_analyzer_table_type_detection_unknown() { + let objects = vec![ + ObjectInfo { + key: "part-00000.parquet".to_string(), + size: 1024, + last_modified: None, + etag: None, + }, + ObjectInfo { + key: "part-00001.parquet".to_string(), + size: 2048, + last_modified: None, + etag: None, + }, + ]; + + // Check for neither Delta Lake nor Iceberg files + let has_delta_log = objects + .iter() + .any(|obj| obj.key.contains("_delta_log/") && obj.key.ends_with(".json")); + let has_iceberg_metadata = objects.iter().any(|obj| obj.key.ends_with("metadata.json")); + + assert!(!has_delta_log, "Should not detect Delta Lake files"); + assert!(!has_iceberg_metadata, "Should not detect Iceberg files"); + // This should be unknown + } + + #[test] + fn test_health_analyzer_storage_client_clone() { + // Test that StorageClient can be cloned // This is important for the HealthAnalyzer implementation let bucket = "test-bucket".to_string(); let prefix = "test-prefix".to_string(); - // In a real test, we'd create an actual S3ClientWrapper and test cloning + // In a real test, we'd create an actual StorageClient and test cloning // For now, we'll test the concept let bucket_clone = bucket.clone(); let prefix_clone = prefix.clone(); @@ -246,4 +399,172 @@ mod tests { assert_eq!(bucket, bucket_clone); assert_eq!(prefix, prefix_clone); } + + #[test] + fn test_health_analyzer_error_handling() { + let invalid_s3_path = "not-a-valid-s3-path"; + let valid_s3_path = "s3://bucket/table/"; + + // Test that invalid paths are handled appropriately + assert!(!invalid_s3_path.starts_with("s3://")); + assert!(valid_s3_path.starts_with("s3://")); + } + + #[test] + fn test_health_analyzer_async_creation() { + // Test that the async creation method signature is correct + let storage_path = "s3://test-bucket/test-table/".to_string(); + let aws_access_key_id = Some("AKIAIOSFODNN7EXAMPLE".to_string()); + let aws_secret_access_key = Some("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string()); + let aws_region = Some("us-west-2".to_string()); + let gcs_service_account_key: Option = None; + + // Verify parameter types match expected signature + assert!(!storage_path.is_empty()); + assert!(aws_access_key_id.is_some()); + assert!(aws_secret_access_key.is_some()); + assert!(aws_region.is_some()); + assert!(gcs_service_account_key.is_none()); + } + + #[test] + fn test_health_analyzer_analyze_delta_lake_signature() { + // Test that the analyze_delta_lake method signature is correct + // This would be an async method that returns PyResult + let expected_return_type = "PyResult"; + assert_eq!(expected_return_type, "PyResult"); + } + + #[test] + fn test_health_analyzer_analyze_iceberg_signature() { + // Test that the analyze_iceberg method signature is correct + // This would be an async method that returns PyResult + let expected_return_type = "PyResult"; + assert_eq!(expected_return_type, "PyResult"); + } + + #[test] + fn test_health_analyzer_list_objects_for_detection_signature() { + // Test that the list_objects_for_detection method signature is correct + // This would be an async method that returns PyResult> + let expected_return_type = "PyResult>"; + assert_eq!(expected_return_type, "PyResult>"); + } + + #[test] + fn test_health_analyzer_pyclass_attributes() { + // Test that HealthAnalyzer has the correct PyClass attributes + let struct_name = "HealthAnalyzer"; + assert_eq!(struct_name, "HealthAnalyzer"); + + // In a real test, we'd verify the #[pyclass] attribute + // and that it implements the correct methods + } + + #[test] + fn test_health_analyzer_pymethods_attributes() { + // Test that HealthAnalyzer has the correct PyMethods attributes + let impl_name = "HealthAnalyzer"; + assert_eq!(impl_name, "HealthAnalyzer"); + + // In a real test, we'd verify the #[pymethods] attribute + // and that it implements the correct methods + } + + #[test] + fn test_health_analyzer_internal_methods() { + // Test that internal methods are properly implemented + let internal_methods = vec![ + "create_async", + "analyze_delta_lake", + "analyze_iceberg", + "list_objects_for_detection", + ]; + + for method in internal_methods { + assert!( + !method.is_empty(), + "Method name should not be empty: {}", + method + ); + } + } + + #[test] + fn test_health_analyzer_public_methods() { + // Test that public methods are properly implemented + let public_methods = vec!["get_table_info"]; + + for method in public_methods { + assert!( + !method.is_empty(), + "Method name should not be empty: {}", + method + ); + } + } + + #[test] + fn test_health_analyzer_error_types() { + // Test that appropriate error types are used + let error_types = vec!["PyRuntimeError", "PyValueError"]; + + for error_type in error_types { + assert!( + !error_type.is_empty(), + "Error type should not be empty: {}", + error_type + ); + } + } + + #[test] + fn test_health_analyzer_storage_client_dependency() { + // Test that HealthAnalyzer properly depends on StorageClient + let dependency = "StorageClient"; + assert_eq!(dependency, "StorageClient"); + + // In a real test, we'd verify that HealthAnalyzer uses StorageClient + // and that the dependency is properly injected + } + + #[test] + fn test_health_analyzer_health_report_dependency() { + // Test that HealthAnalyzer properly depends on HealthReport + let dependency = "HealthReport"; + assert_eq!(dependency, "HealthReport"); + + // In a real test, we'd verify that HealthAnalyzer returns HealthReport + // and that the dependency is properly used + } + + #[test] + fn test_health_analyzer_async_runtime() { + // Test that HealthAnalyzer properly uses async runtime + let runtime = "tokio"; + assert_eq!(runtime, "tokio"); + + // In a real test, we'd verify that HealthAnalyzer uses tokio runtime + // and that async methods are properly implemented + } + + #[test] + fn test_health_analyzer_pyresult_usage() { + // Test that HealthAnalyzer properly uses PyResult for error handling + let result_type = "PyResult"; + assert_eq!(result_type, "PyResult"); + + // In a real test, we'd verify that HealthAnalyzer methods return PyResult + // and that errors are properly converted to Python exceptions + } + + #[test] + fn test_health_analyzer_anyhow_usage() { + // Test that HealthAnalyzer properly uses anyhow for error handling + let error_crate = "anyhow"; + assert_eq!(error_crate, "anyhow"); + + // In a real test, we'd verify that HealthAnalyzer uses anyhow::Result + // and that errors are properly converted to PyResult + } } diff --git a/src/health_analyzer_tests.rs b/src/health_analyzer_tests.rs deleted file mode 100644 index 939b89b..0000000 --- a/src/health_analyzer_tests.rs +++ /dev/null @@ -1,419 +0,0 @@ -#[cfg(test)] -mod tests { - use super::*; - use crate::health_analyzer::*; - use crate::s3_client::*; - use crate::types::*; - - #[test] - fn test_health_analyzer_get_table_info() { - // This test would require a mock S3ClientWrapper - // For now, we'll test the concept - let bucket = "test-bucket".to_string(); - let prefix = "test-prefix".to_string(); - - // In a real test, we'd create a mock HealthAnalyzer - // and verify that get_table_info returns the correct values - assert_eq!(bucket, "test-bucket"); - assert_eq!(prefix, "test-prefix"); - } - - #[test] - fn test_health_analyzer_creation_parameters() { - let s3_path = "s3://test-bucket/test-table/"; - let aws_access_key_id = Some("AKIAIOSFODNN7EXAMPLE".to_string()); - let aws_secret_access_key = Some("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string()); - let aws_region = Some("us-west-2".to_string()); - - // Test parameter validation - assert!(aws_access_key_id.is_some()); - assert!(aws_secret_access_key.is_some()); - assert!(aws_region.is_some()); - assert!(s3_path.starts_with("s3://")); - } - - #[test] - fn test_health_analyzer_creation_without_credentials() { - let s3_path = "s3://test-bucket/test-table/"; - let aws_access_key_id = None; - let aws_secret_access_key = None; - let aws_region = Some("us-west-2".to_string()); - - // Test parameter validation for IAM role usage - assert!(aws_access_key_id.is_none()); - assert!(aws_secret_access_key.is_none()); - assert!(aws_region.is_some()); - assert!(s3_path.starts_with("s3://")); - } - - #[test] - fn test_health_analyzer_creation_without_region() { - let s3_path = "s3://test-bucket/test-table/"; - let aws_access_key_id = Some("AKIAIOSFODNN7EXAMPLE".to_string()); - let aws_secret_access_key = Some("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string()); - let aws_region = None; - - // Test parameter validation for default region - assert!(aws_access_key_id.is_some()); - assert!(aws_secret_access_key.is_some()); - assert!(aws_region.is_none()); - assert!(s3_path.starts_with("s3://")); - } - - #[test] - fn test_health_analyzer_s3_path_validation() { - let valid_paths = vec![ - "s3://bucket/table/", - "s3://my-bucket/my-table/", - "s3://bucket.with.dots/table/", - "s3://bucket/path/to/table/", - ]; - - for path in valid_paths { - assert!(path.starts_with("s3://"), "Invalid S3 path: {}", path); - assert!(path.contains("/"), "S3 path should contain path separator: {}", path); - } - } - - #[test] - fn test_health_analyzer_s3_path_validation_invalid() { - let invalid_paths = vec![ - "https://bucket/table/", - "ftp://bucket/table/", - "not-a-url", - "", - "s3://", - "s3:///", - ]; - - for path in invalid_paths { - if path.is_empty() { - continue; // Skip empty string test - } - assert!(!path.starts_with("s3://") || path == "s3://" || path == "s3:///", - "Should be invalid S3 path: {}", path); - } - } - - #[test] - fn test_health_analyzer_aws_region_validation() { - let valid_regions = vec![ - "us-east-1", - "us-west-2", - "eu-west-1", - "ap-southeast-1", - "ca-central-1", - ]; - - for region in valid_regions { - assert!(!region.is_empty(), "Region should not be empty"); - assert!(region.contains("-"), "Region should contain dash: {}", region); - } - } - - #[test] - fn test_health_analyzer_aws_credentials_validation() { - let valid_access_key = "AKIAIOSFODNN7EXAMPLE"; - let valid_secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - - assert!(valid_access_key.starts_with("AKIA"), "Access key should start with AKIA"); - assert!(valid_secret_key.len() >= 20, "Secret key should be at least 20 characters"); - assert!(!valid_access_key.contains(" "), "Access key should not contain spaces"); - assert!(!valid_secret_key.contains(" "), "Secret key should not contain spaces"); - } - - #[test] - fn test_health_analyzer_table_type_detection_delta() { - let objects = vec![ - ObjectInfo { - key: "part-00000.parquet".to_string(), - size: 1024, - last_modified: None, - etag: None, - }, - ObjectInfo { - key: "_delta_log/00000000000000000000.json".to_string(), - size: 2048, - last_modified: None, - etag: None, - }, - ObjectInfo { - key: "_delta_log/00000000000000000001.json".to_string(), - size: 1024, - last_modified: None, - etag: None, - }, - ]; - - // Check for Delta Lake characteristic files - let has_delta_log = objects.iter().any(|obj| obj.key.contains("_delta_log/") && obj.key.ends_with(".json")); - let has_iceberg_metadata = objects.iter().any(|obj| obj.key.ends_with("metadata.json")); - - assert!(has_delta_log, "Should detect Delta Lake files"); - assert!(!has_iceberg_metadata, "Should not detect Iceberg files"); - } - - #[test] - fn test_health_analyzer_table_type_detection_iceberg() { - let objects = vec![ - ObjectInfo { - key: "data/00000-0-00000000000000000000-00000000000000000000.parquet".to_string(), - size: 1024, - last_modified: None, - etag: None, - }, - ObjectInfo { - key: "metadata/00000-00000000000000000000.metadata.json".to_string(), - size: 2048, - last_modified: None, - etag: None, - }, - ObjectInfo { - key: "metadata/snap-00000000000000000000-1-00000000000000000000.avro".to_string(), - size: 1024, - last_modified: None, - etag: None, - }, - ]; - - // Check for Iceberg characteristic files - let has_delta_log = objects.iter().any(|obj| obj.key.contains("_delta_log/") && obj.key.ends_with(".json")); - let has_iceberg_metadata = objects.iter().any(|obj| obj.key.ends_with("metadata.json")); - - assert!(!has_delta_log, "Should not detect Delta Lake files"); - assert!(has_iceberg_metadata, "Should detect Iceberg files"); - } - - #[test] - fn test_health_analyzer_table_type_detection_ambiguous() { - let objects = vec![ - ObjectInfo { - key: "part-00000.parquet".to_string(), - size: 1024, - last_modified: None, - etag: None, - }, - ObjectInfo { - key: "_delta_log/00000000000000000000.json".to_string(), - size: 2048, - last_modified: None, - etag: None, - }, - ObjectInfo { - key: "metadata/00000-00000000000000000000.metadata.json".to_string(), - size: 1024, - last_modified: None, - etag: None, - }, - ]; - - // Check for both Delta Lake and Iceberg files - let has_delta_log = objects.iter().any(|obj| obj.key.contains("_delta_log/") && obj.key.ends_with(".json")); - let has_iceberg_metadata = objects.iter().any(|obj| obj.key.ends_with("metadata.json")); - - assert!(has_delta_log, "Should detect Delta Lake files"); - assert!(has_iceberg_metadata, "Should detect Iceberg files"); - // This should be ambiguous - } - - #[test] - fn test_health_analyzer_table_type_detection_unknown() { - let objects = vec![ - ObjectInfo { - key: "part-00000.parquet".to_string(), - size: 1024, - last_modified: None, - etag: None, - }, - ObjectInfo { - key: "part-00001.parquet".to_string(), - size: 2048, - last_modified: None, - etag: None, - }, - ]; - - // Check for neither Delta Lake nor Iceberg files - let has_delta_log = objects.iter().any(|obj| obj.key.contains("_delta_log/") && obj.key.ends_with(".json")); - let has_iceberg_metadata = objects.iter().any(|obj| obj.key.ends_with("metadata.json")); - - assert!(!has_delta_log, "Should not detect Delta Lake files"); - assert!(!has_iceberg_metadata, "Should not detect Iceberg files"); - // This should be unknown - } - - #[test] - fn test_health_analyzer_s3_client_wrapper_clone() { - // Test that S3ClientWrapper can be cloned - // This is important for the HealthAnalyzer implementation - let bucket = "test-bucket".to_string(); - let prefix = "test-prefix".to_string(); - - // In a real test, we'd create an actual S3ClientWrapper and test cloning - // For now, we'll test the concept - let bucket_clone = bucket.clone(); - let prefix_clone = prefix.clone(); - - assert_eq!(bucket, bucket_clone); - assert_eq!(prefix, prefix_clone); - } - - #[test] - fn test_health_analyzer_error_handling() { - let invalid_s3_path = "not-a-valid-s3-path"; - let valid_s3_path = "s3://bucket/table/"; - - // Test that invalid paths are handled appropriately - assert!(!invalid_s3_path.starts_with("s3://")); - assert!(valid_s3_path.starts_with("s3://")); - } - - #[test] - fn test_health_analyzer_async_creation() { - // Test that the async creation method signature is correct - let s3_path = "s3://test-bucket/test-table/".to_string(); - let aws_access_key_id = Some("AKIAIOSFODNN7EXAMPLE".to_string()); - let aws_secret_access_key = Some("wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY".to_string()); - let aws_region = Some("us-west-2".to_string()); - - // Verify parameter types match expected signature - assert!(s3_path.is_string()); - assert!(aws_access_key_id.is_some()); - assert!(aws_secret_access_key.is_some()); - assert!(aws_region.is_some()); - } - - #[test] - fn test_health_analyzer_analyze_delta_lake_signature() { - // Test that the analyze_delta_lake method signature is correct - // This would be an async method that returns PyResult - let expected_return_type = "PyResult"; - assert_eq!(expected_return_type, "PyResult"); - } - - #[test] - fn test_health_analyzer_analyze_iceberg_signature() { - // Test that the analyze_iceberg method signature is correct - // This would be an async method that returns PyResult - let expected_return_type = "PyResult"; - assert_eq!(expected_return_type, "PyResult"); - } - - #[test] - fn test_health_analyzer_list_objects_for_detection_signature() { - // Test that the list_objects_for_detection method signature is correct - // This would be an async method that returns PyResult> - let expected_return_type = "PyResult>"; - assert_eq!(expected_return_type, "PyResult>"); - } - - #[test] - fn test_health_analyzer_pyclass_attributes() { - // Test that HealthAnalyzer has the correct PyClass attributes - let struct_name = "HealthAnalyzer"; - assert_eq!(struct_name, "HealthAnalyzer"); - - // In a real test, we'd verify the #[pyclass] attribute - // and that it implements the correct methods - } - - #[test] - fn test_health_analyzer_pymethods_attributes() { - // Test that HealthAnalyzer has the correct PyMethods attributes - let impl_name = "HealthAnalyzer"; - assert_eq!(impl_name, "HealthAnalyzer"); - - // In a real test, we'd verify the #[pymethods] attribute - // and that it implements the correct methods - } - - #[test] - fn test_health_analyzer_internal_methods() { - // Test that internal methods are properly implemented - let internal_methods = vec![ - "create_async", - "analyze_delta_lake", - "analyze_iceberg", - "list_objects_for_detection", - ]; - - for method in internal_methods { - assert!(!method.is_empty(), "Method name should not be empty: {}", method); - } - } - - #[test] - fn test_health_analyzer_public_methods() { - // Test that public methods are properly implemented - let public_methods = vec![ - "get_table_info", - ]; - - for method in public_methods { - assert!(!method.is_empty(), "Method name should not be empty: {}", method); - } - } - - #[test] - fn test_health_analyzer_error_types() { - // Test that appropriate error types are used - let error_types = vec![ - "PyRuntimeError", - "PyValueError", - ]; - - for error_type in error_types { - assert!(!error_type.is_empty(), "Error type should not be empty: {}", error_type); - } - } - - #[test] - fn test_health_analyzer_s3_client_wrapper_dependency() { - // Test that HealthAnalyzer properly depends on S3ClientWrapper - let dependency = "S3ClientWrapper"; - assert_eq!(dependency, "S3ClientWrapper"); - - // In a real test, we'd verify that HealthAnalyzer uses S3ClientWrapper - // and that the dependency is properly injected - } - - #[test] - fn test_health_analyzer_health_report_dependency() { - // Test that HealthAnalyzer properly depends on HealthReport - let dependency = "HealthReport"; - assert_eq!(dependency, "HealthReport"); - - // In a real test, we'd verify that HealthAnalyzer returns HealthReport - // and that the dependency is properly used - } - - #[test] - fn test_health_analyzer_async_runtime() { - // Test that HealthAnalyzer properly uses async runtime - let runtime = "tokio"; - assert_eq!(runtime, "tokio"); - - // In a real test, we'd verify that HealthAnalyzer uses tokio runtime - // and that async methods are properly implemented - } - - #[test] - fn test_health_analyzer_pyresult_usage() { - // Test that HealthAnalyzer properly uses PyResult for error handling - let result_type = "PyResult"; - assert_eq!(result_type, "PyResult"); - - // In a real test, we'd verify that HealthAnalyzer methods return PyResult - // and that errors are properly converted to Python exceptions - } - - #[test] - fn test_health_analyzer_anyhow_usage() { - // Test that HealthAnalyzer properly uses anyhow for error handling - let error_crate = "anyhow"; - assert_eq!(error_crate, "anyhow"); - - // In a real test, we'd verify that HealthAnalyzer uses anyhow::Result - // and that errors are properly converted to PyResult - } -} diff --git a/src/iceberg.rs b/src/iceberg.rs index 15b0ab8..b8a127c 100644 --- a/src/iceberg.rs +++ b/src/iceberg.rs @@ -1,4 +1,4 @@ -use crate::s3_client::S3ClientWrapper; +use crate::storage_client::StorageClient; use crate::types::*; use anyhow::Result; use serde_json::Value; @@ -14,28 +14,28 @@ struct SchemaChange { } pub struct IcebergAnalyzer { - s3_client: S3ClientWrapper, + storage_client: StorageClient, } impl IcebergAnalyzer { - pub fn new(s3_client: S3ClientWrapper) -> Self { - Self { s3_client } + pub fn new(storage_client: StorageClient) -> Self { + Self { storage_client } } pub async fn analyze(&self) -> Result { let mut report = HealthReport::new( format!( - "s3://{}/{}", - self.s3_client.get_bucket(), - self.s3_client.get_prefix() + "{}/{}", + self.storage_client.get_bucket(), + self.storage_client.get_prefix() ), "iceberg".to_string(), ); // List all files in the Iceberg table directory let all_objects = self - .s3_client - .list_objects(self.s3_client.get_prefix()) + .storage_client + .list_objects(self.storage_client.get_prefix()) .await?; // Find the current metadata.json file @@ -59,7 +59,7 @@ impl IcebergAnalyzer { // Find unreferenced files let referenced_set: HashSet = referenced_files.into_iter().collect(); for file in &data_files { - let file_path = format!("{}/{}", self.s3_client.get_prefix(), file.key); + let file_path = format!("{}/{}", self.storage_client.get_prefix(), file.key); if !referenced_set.contains(&file_path) { metrics.unreferenced_files.push(FileInfo { path: file_path, @@ -90,7 +90,7 @@ impl IcebergAnalyzer { // Calculate additional health metrics metrics.calculate_data_skew(); - let metadata_files_owned: Vec = + let metadata_files_owned: Vec = metadata_files.iter().map(|f| (*f).clone()).collect(); metrics.calculate_metadata_health(&metadata_files_owned); metrics.calculate_snapshot_health(metadata_files.len()); // Simplified: use metadata file count as snapshot count @@ -127,10 +127,10 @@ impl IcebergAnalyzer { fn find_current_metadata<'a>( &self, - objects: &'a [crate::s3_client::ObjectInfo], - ) -> Result<&'a crate::s3_client::ObjectInfo> { + objects: &'a [crate::storage_client::ObjectInfo], + ) -> Result<&'a crate::storage_client::ObjectInfo> { // Find the most recent metadata.json file - let metadata_files: Vec<&crate::s3_client::ObjectInfo> = objects + let metadata_files: Vec<&crate::storage_client::ObjectInfo> = objects .iter() .filter(|obj| obj.key.ends_with("metadata.json")) .collect(); @@ -151,8 +151,11 @@ impl IcebergAnalyzer { Ok(sorted_files[0]) } - async fn load_metadata(&self, metadata_file: &crate::s3_client::ObjectInfo) -> Result { - let content = self.s3_client.get_object(&metadata_file.key).await?; + async fn load_metadata( + &self, + metadata_file: &crate::storage_client::ObjectInfo, + ) -> Result { + let content = self.storage_client.get_object(&metadata_file.key).await?; let metadata: Value = serde_json::from_slice(&content)?; Ok(metadata) } @@ -162,7 +165,7 @@ impl IcebergAnalyzer { if let Some(manifest_list_path) = metadata.get("manifest-list") { if let Some(path) = manifest_list_path.as_str() { - let content = self.s3_client.get_object(path).await?; + let content = self.storage_client.get_object(path).await?; let manifest_list_json: Value = serde_json::from_slice(&content)?; if let Some(manifests) = manifest_list_json.get("manifests") { @@ -186,7 +189,7 @@ impl IcebergAnalyzer { let mut referenced_files = Vec::new(); for manifest_path in manifest_list { - let content = self.s3_client.get_object(manifest_path).await?; + let content = self.storage_client.get_object(manifest_path).await?; let manifest: Value = serde_json::from_slice(&content)?; if let Some(entries) = manifest.get("entries") { @@ -209,10 +212,10 @@ impl IcebergAnalyzer { fn categorize_files<'a>( &self, - objects: &'a [crate::s3_client::ObjectInfo], + objects: &'a [crate::storage_client::ObjectInfo], ) -> Result<( - Vec<&'a crate::s3_client::ObjectInfo>, - Vec<&'a crate::s3_client::ObjectInfo>, + Vec<&'a crate::storage_client::ObjectInfo>, + Vec<&'a crate::storage_client::ObjectInfo>, )> { let mut data_files = Vec::new(); let mut metadata_files = Vec::new(); @@ -230,7 +233,7 @@ impl IcebergAnalyzer { fn analyze_partitioning_and_clustering( &self, - data_files: &[&crate::s3_client::ObjectInfo], + data_files: &[&crate::storage_client::ObjectInfo], metadata: &Value, metrics: &mut HealthMetrics, ) -> Result<()> { @@ -281,7 +284,7 @@ impl IcebergAnalyzer { partition_info.file_count += 1; partition_info.total_size_bytes += file.size as u64; partition_info.files.push(FileInfo { - path: format!("{}/{}", self.s3_client.get_prefix(), file.key), + path: format!("{}/{}", self.storage_client.get_prefix(), file.key), size_bytes: file.size as u64, last_modified: file.last_modified.clone(), is_referenced: true, // We'll update this later @@ -344,7 +347,7 @@ impl IcebergAnalyzer { fn calculate_file_size_distribution( &self, - data_files: &[&crate::s3_client::ObjectInfo], + data_files: &[&crate::storage_client::ObjectInfo], metrics: &mut HealthMetrics, ) { for file in data_files { @@ -616,7 +619,7 @@ impl IcebergAnalyzer { // Analyze manifest files for deletion vectors for manifest_path in manifest_list { // Download and analyze manifest file - let manifest_content = self.s3_client.get_object(manifest_path).await?; + let manifest_content = self.storage_client.get_object(manifest_path).await?; let manifest_json: Value = serde_json::from_slice(&manifest_content)?; // Look for deletion files in manifest @@ -704,7 +707,7 @@ impl IcebergAnalyzer { async fn analyze_schema_evolution( &self, - metadata_files: &[&crate::s3_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result> { let mut schema_changes = Vec::new(); let mut current_version = 0; @@ -722,7 +725,7 @@ impl IcebergAnalyzer { for metadata_file in &sorted_files { // Try to get the metadata file, but skip if it doesn't exist (race condition) - let content = match self.s3_client.get_object(&metadata_file.key).await { + let content = match self.storage_client.get_object(&metadata_file.key).await { Ok(c) => c, Err(_) => continue, }; @@ -952,7 +955,7 @@ impl IcebergAnalyzer { async fn analyze_time_travel( &self, - metadata_files: &[&crate::s3_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result> { let mut total_snapshots = 0; let mut total_historical_size = 0u64; @@ -962,7 +965,7 @@ impl IcebergAnalyzer { // Analyze metadata files for time travel storage for metadata_file in metadata_files { // Try to get the metadata file, but skip if it doesn't exist (race condition) - let content = match self.s3_client.get_object(&metadata_file.key).await { + let content = match self.storage_client.get_object(&metadata_file.key).await { Ok(c) => c, Err(_) => continue, }; @@ -1118,7 +1121,7 @@ impl IcebergAnalyzer { async fn analyze_table_constraints( &self, - metadata_files: &[&crate::s3_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result> { let mut total_constraints = 0; let mut check_constraints = 0; @@ -1129,7 +1132,7 @@ impl IcebergAnalyzer { // Analyze metadata files for constraint information for metadata_file in metadata_files { // Try to get the metadata file, but skip if it doesn't exist (race condition) - let content = match self.s3_client.get_object(&metadata_file.key).await { + let content = match self.storage_client.get_object(&metadata_file.key).await { Ok(c) => c, Err(_) => continue, }; @@ -1278,8 +1281,8 @@ impl IcebergAnalyzer { async fn analyze_file_compaction( &self, - data_files: &[&crate::s3_client::ObjectInfo], - metadata_files: &[&crate::s3_client::ObjectInfo], + data_files: &[&crate::storage_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result> { let mut small_files_count = 0; let mut small_files_size = 0u64; @@ -1360,7 +1363,7 @@ impl IcebergAnalyzer { fn calculate_recommended_target_size( &self, - data_files: &[&crate::s3_client::ObjectInfo], + data_files: &[&crate::storage_client::ObjectInfo], ) -> u64 { if data_files.is_empty() { return 128 * 1024 * 1024; // 128MB default @@ -1393,12 +1396,12 @@ impl IcebergAnalyzer { async fn analyze_iceberg_z_order_opportunity( &self, - metadata_files: &[&crate::s3_client::ObjectInfo], + metadata_files: &[&crate::storage_client::ObjectInfo], ) -> Result<(bool, Vec)> { // Look for sort order information that could benefit from Z-ordering for metadata_file in metadata_files { // Try to get the metadata file, but skip if it doesn't exist (race condition) - let content = match self.s3_client.get_object(&metadata_file.key).await { + let content = match self.storage_client.get_object(&metadata_file.key).await { Ok(c) => c, Err(_) => continue, }; diff --git a/src/lib.rs b/src/lib.rs index 08e6b02..97c7df1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,14 +3,14 @@ use pyo3::prelude::*; mod delta_lake; mod health_analyzer; mod iceberg; -mod s3_client; +mod storage_client; mod types; use health_analyzer::HealthAnalyzer; /// A Python module implemented in Rust for analyzing data lake health #[pymodule] -fn drainage(_py: Python, m: &PyModule) -> PyResult<()> { +fn drainage(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(analyze_delta_lake, m)?)?; m.add_function(wrap_pyfunction!(analyze_iceberg, m)?)?; m.add_function(wrap_pyfunction!(analyze_table, m)?)?; @@ -21,18 +21,20 @@ fn drainage(_py: Python, m: &PyModule) -> PyResult<()> { /// Analyze Delta Lake table health #[pyfunction] fn analyze_delta_lake( - s3_path: String, + storage_path: String, aws_access_key_id: Option, aws_secret_access_key: Option, aws_region: Option, + gcs_service_account_key: Option, ) -> PyResult { let rt = tokio::runtime::Runtime::new()?; rt.block_on(async { let analyzer = HealthAnalyzer::create_async( - s3_path, + storage_path, aws_access_key_id, aws_secret_access_key, aws_region, + gcs_service_account_key, ) .await?; analyzer.analyze_delta_lake().await @@ -42,18 +44,20 @@ fn analyze_delta_lake( /// Analyze Apache Iceberg table health #[pyfunction] fn analyze_iceberg( - s3_path: String, + storage_path: String, aws_access_key_id: Option, aws_secret_access_key: Option, aws_region: Option, + gcs_service_account_key: Option, ) -> PyResult { let rt = tokio::runtime::Runtime::new()?; rt.block_on(async { let analyzer = HealthAnalyzer::create_async( - s3_path, + storage_path, aws_access_key_id, aws_secret_access_key, aws_region, + gcs_service_account_key, ) .await?; analyzer.analyze_iceberg().await @@ -63,15 +67,16 @@ fn analyze_iceberg( /// Analyze table health with automatic table type detection #[pyfunction] fn analyze_table( - s3_path: String, + storage_path: String, table_type: Option, aws_access_key_id: Option, aws_secret_access_key: Option, aws_region: Option, + gcs_service_account_key: Option, ) -> PyResult { let rt = tokio::runtime::Runtime::new()?; rt.block_on(async { - let analyzer = HealthAnalyzer::create_async(s3_path.clone(), aws_access_key_id, aws_secret_access_key, aws_region).await?; + let analyzer = HealthAnalyzer::create_async(storage_path.clone(), aws_access_key_id, aws_secret_access_key, aws_region, gcs_service_account_key).await?; // If table type is specified, use it directly if let Some(ref ttype) = table_type { match ttype.to_lowercase().as_str() { diff --git a/src/s3_client_tests.rs b/src/s3_client_tests.rs deleted file mode 100644 index 028aabe..0000000 --- a/src/s3_client_tests.rs +++ /dev/null @@ -1,400 +0,0 @@ -#[cfg(test)] -mod tests { - use super::*; - use crate::s3_client::*; - use url::Url; - - #[test] - fn test_object_info_creation() { - let object_info = ObjectInfo { - key: "test/file.parquet".to_string(), - size: 1024, - last_modified: Some("2023-01-01T00:00:00Z".to_string()), - etag: Some("etag123".to_string()), - }; - - assert_eq!(object_info.key, "test/file.parquet"); - assert_eq!(object_info.size, 1024); - assert_eq!(object_info.last_modified, Some("2023-01-01T00:00:00Z".to_string())); - assert_eq!(object_info.etag, Some("etag123".to_string())); - } - - #[test] - fn test_object_info_clone() { - let object_info = ObjectInfo { - key: "test/file.parquet".to_string(), - size: 1024, - last_modified: Some("2023-01-01T00:00:00Z".to_string()), - etag: Some("etag123".to_string()), - }; - - let cloned = object_info.clone(); - assert_eq!(cloned.key, object_info.key); - assert_eq!(cloned.size, object_info.size); - assert_eq!(cloned.last_modified, object_info.last_modified); - assert_eq!(cloned.etag, object_info.etag); - } - - #[test] - fn test_object_info_debug() { - let object_info = ObjectInfo { - key: "test/file.parquet".to_string(), - size: 1024, - last_modified: Some("2023-01-01T00:00:00Z".to_string()), - etag: Some("etag123".to_string()), - }; - - let debug_str = format!("{:?}", object_info); - assert!(debug_str.contains("test/file.parquet")); - assert!(debug_str.contains("1024")); - } - - #[test] - fn test_s3_url_parsing_valid() { - let s3_path = "s3://my-bucket/my-table/"; - let url = Url::parse(s3_path).unwrap(); - - assert_eq!(url.scheme(), "s3"); - assert_eq!(url.host_str(), Some("my-bucket")); - assert_eq!(url.path(), "/my-table/"); - } - - #[test] - fn test_s3_url_parsing_with_prefix() { - let s3_path = "s3://my-bucket/my-table/year=2023/month=01/"; - let url = Url::parse(s3_path).unwrap(); - - assert_eq!(url.scheme(), "s3"); - assert_eq!(url.host_str(), Some("my-bucket")); - assert_eq!(url.path(), "/my-table/year=2023/month=01/"); - } - - #[test] - fn test_s3_url_parsing_invalid() { - let invalid_path = "not-a-url"; - let result = Url::parse(invalid_path); - assert!(result.is_err()); - } - - #[test] - fn test_s3_url_parsing_missing_bucket() { - let s3_path = "s3:///my-table/"; - let url = Url::parse(s3_path).unwrap(); - - // This should be valid URL parsing but bucket will be empty - assert_eq!(url.scheme(), "s3"); - assert_eq!(url.host_str(), None); - } - - #[test] - fn test_s3_path_components_extraction() { - let s3_path = "s3://my-bucket/my-table/year=2023/month=01/"; - let url = Url::parse(s3_path).unwrap(); - - let bucket = url.host_str().unwrap().to_string(); - let prefix = url.path().trim_start_matches('/').to_string(); - - assert_eq!(bucket, "my-bucket"); - assert_eq!(prefix, "my-table/year=2023/month=01/"); - } - - #[test] - fn test_s3_path_components_extraction_no_trailing_slash() { - let s3_path = "s3://my-bucket/my-table"; - let url = Url::parse(s3_path).unwrap(); - - let bucket = url.host_str().unwrap().to_string(); - let prefix = url.path().trim_start_matches('/').to_string(); - - assert_eq!(bucket, "my-bucket"); - assert_eq!(prefix, "my-table"); - } - - #[test] - fn test_s3_path_components_extraction_root_bucket() { - let s3_path = "s3://my-bucket/"; - let url = Url::parse(s3_path).unwrap(); - - let bucket = url.host_str().unwrap().to_string(); - let prefix = url.path().trim_start_matches('/').to_string(); - - assert_eq!(bucket, "my-bucket"); - assert_eq!(prefix, ""); - } - - #[test] - fn test_s3_path_components_extraction_nested_path() { - let s3_path = "s3://my-bucket/data/lake/tables/my-table/"; - let url = Url::parse(s3_path).unwrap(); - - let bucket = url.host_str().unwrap().to_string(); - let prefix = url.path().trim_start_matches('/').to_string(); - - assert_eq!(bucket, "my-bucket"); - assert_eq!(prefix, "data/lake/tables/my-table/"); - } - - #[test] - fn test_aws_region_creation() { - let region_str = "us-west-2"; - let region = aws_sdk_s3::config::Region::new(region_str); - - assert_eq!(region.as_ref(), "us-west-2"); - } - - #[test] - fn test_aws_region_creation_eu_region() { - let region_str = "eu-west-1"; - let region = aws_sdk_s3::config::Region::new(region_str); - - assert_eq!(region.as_ref(), "eu-west-1"); - } - - #[test] - fn test_aws_region_creation_ap_region() { - let region_str = "ap-southeast-1"; - let region = aws_sdk_s3::config::Region::new(region_str); - - assert_eq!(region.as_ref(), "ap-southeast-1"); - } - - #[test] - fn test_aws_credentials_creation() { - let access_key = "AKIAIOSFODNN7EXAMPLE"; - let secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - - let creds = aws_sdk_s3::config::Credentials::new( - access_key, - secret_key, - None, - None, - "drainage" - ); - - assert_eq!(creds.access_key_id(), access_key); - assert_eq!(creds.secret_access_key(), secret_key); - assert_eq!(creds.session_token(), None); - assert_eq!(creds.expiry(), None); - assert_eq!(creds.provider_name(), "drainage"); - } - - #[test] - fn test_aws_credentials_creation_with_session_token() { - let access_key = "AKIAIOSFODNN7EXAMPLE"; - let secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - let session_token = "session-token-example"; - - let creds = aws_sdk_s3::config::Credentials::new( - access_key, - secret_key, - Some(session_token), - None, - "drainage" - ); - - assert_eq!(creds.access_key_id(), access_key); - assert_eq!(creds.secret_access_key(), secret_key); - assert_eq!(creds.session_token(), Some(session_token)); - assert_eq!(creds.expiry(), None); - assert_eq!(creds.provider_name(), "drainage"); - } - - #[test] - fn test_aws_credentials_creation_with_expiry() { - use std::time::{Duration, SystemTime, UNIX_EPOCH}; - - let access_key = "AKIAIOSFODNN7EXAMPLE"; - let secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - let expiry = SystemTime::now() + Duration::from_secs(3600); - - let creds = aws_sdk_s3::config::Credentials::new( - access_key, - secret_key, - None, - Some(expiry), - "drainage" - ); - - assert_eq!(creds.access_key_id(), access_key); - assert_eq!(creds.secret_access_key(), secret_key); - assert_eq!(creds.session_token(), None); - assert_eq!(creds.expiry(), Some(expiry)); - assert_eq!(creds.provider_name(), "drainage"); - } - - #[test] - fn test_s3_client_wrapper_getters() { - // This test would require actual S3 client creation, which is complex in unit tests - // We'll test the getter methods conceptually - let bucket = "test-bucket".to_string(); - let prefix = "test-prefix".to_string(); - - // In a real test, we'd create a mock S3ClientWrapper - // For now, we'll just test the string operations - assert_eq!(bucket, "test-bucket"); - assert_eq!(prefix, "test-prefix"); - } - - #[test] - fn test_s3_path_validation() { - let valid_paths = vec![ - "s3://bucket/", - "s3://bucket/path/", - "s3://bucket/path/to/table/", - "s3://my-bucket-name/my-table/", - "s3://bucket.with.dots/table/", - ]; - - for path in valid_paths { - let result = Url::parse(path); - assert!(result.is_ok(), "Failed to parse valid S3 path: {}", path); - - let url = result.unwrap(); - assert_eq!(url.scheme(), "s3"); - assert!(url.host_str().is_some(), "Missing bucket in path: {}", path); - } - } - - #[test] - fn test_s3_path_validation_invalid() { - let invalid_paths = vec![ - "https://bucket/", - "ftp://bucket/", - "not-a-url", - "", - "s3://", - "s3:///", - ]; - - for path in invalid_paths { - if path.is_empty() { - continue; // Skip empty string test as it's handled differently - } - - let result = Url::parse(path); - if result.is_ok() { - let url = result.unwrap(); - if url.scheme() != "s3" { - // This is expected for non-s3 URLs - continue; - } - if url.host_str().is_none() { - // This is expected for s3:// or s3:/// - continue; - } - } - } - } - - #[test] - fn test_object_info_optional_fields() { - let object_info_with_all = ObjectInfo { - key: "test/file.parquet".to_string(), - size: 1024, - last_modified: Some("2023-01-01T00:00:00Z".to_string()), - etag: Some("etag123".to_string()), - }; - - let object_info_minimal = ObjectInfo { - key: "test/file.parquet".to_string(), - size: 1024, - last_modified: None, - etag: None, - }; - - assert!(object_info_with_all.last_modified.is_some()); - assert!(object_info_with_all.etag.is_some()); - assert!(object_info_minimal.last_modified.is_none()); - assert!(object_info_minimal.etag.is_none()); - } - - #[test] - fn test_object_info_size_types() { - let small_object = ObjectInfo { - key: "small.parquet".to_string(), - size: 1024, // 1KB - last_modified: None, - etag: None, - }; - - let large_object = ObjectInfo { - key: "large.parquet".to_string(), - size: 1024 * 1024 * 1024, // 1GB - last_modified: None, - etag: None, - }; - - assert_eq!(small_object.size, 1024); - assert_eq!(large_object.size, 1024 * 1024 * 1024); - assert!(large_object.size > small_object.size); - } - - #[test] - fn test_object_info_key_variations() { - let keys = vec![ - "file.parquet", - "path/to/file.parquet", - "deeply/nested/path/to/file.parquet", - "file_with_underscores.parquet", - "file-with-dashes.parquet", - "file.with.dots.parquet", - "file123.parquet", - "UPPERCASE.parquet", - "mixedCase.parquet", - ]; - - for key in keys { - let object_info = ObjectInfo { - key: key.to_string(), - size: 1024, - last_modified: None, - etag: None, - }; - - assert_eq!(object_info.key, key); - } - } - - #[test] - fn test_object_info_etag_variations() { - let etags = vec![ - Some("\"etag123\"".to_string()), - Some("etag123".to_string()), - Some("".to_string()), - None, - ]; - - for etag in etags { - let object_info = ObjectInfo { - key: "test.parquet".to_string(), - size: 1024, - last_modified: None, - etag: etag.clone(), - }; - - assert_eq!(object_info.etag, etag); - } - } - - #[test] - fn test_object_info_last_modified_variations() { - let timestamps = vec![ - Some("2023-01-01T00:00:00Z".to_string()), - Some("2023-12-31T23:59:59Z".to_string()), - Some("2023-01-01T00:00:00.000Z".to_string()), - Some("2023-01-01T00:00:00+00:00".to_string()), - None, - ]; - - for timestamp in timestamps { - let object_info = ObjectInfo { - key: "test.parquet".to_string(), - size: 1024, - last_modified: timestamp.clone(), - etag: None, - }; - - assert_eq!(object_info.last_modified, timestamp); - } - } -} diff --git a/src/s3_client.rs b/src/storage_client.rs similarity index 52% rename from src/s3_client.rs rename to src/storage_client.rs index 567a48b..1a52f4e 100644 --- a/src/s3_client.rs +++ b/src/storage_client.rs @@ -1,108 +1,119 @@ use anyhow::Result; -use aws_config::meta::region::RegionProviderChain; -use aws_sdk_s3::{config::Credentials, config::Region, Client as S3Client}; +use bytes::Bytes; +use futures::stream::StreamExt; +use object_store::{aws::AmazonS3Builder, gcp::GoogleCloudStorageBuilder, ObjectStore}; +use std::sync::Arc; use url::Url; -pub struct S3ClientWrapper { - pub client: S3Client, - pub bucket: String, - pub prefix: String, +pub struct StorageClient { + store: Arc, + bucket: String, + prefix: String, } -impl S3ClientWrapper { +// Implement Clone for StorageClient +impl Clone for StorageClient { + fn clone(&self) -> Self { + Self { + store: Arc::clone(&self.store), + bucket: self.bucket.clone(), + prefix: self.prefix.clone(), + } + } +} + +impl StorageClient { pub async fn new( - s3_path: &str, + storage_path: &str, + // AWS credentials (for s3:// URLs) aws_access_key_id: Option, aws_secret_access_key: Option, aws_region: Option, + // GCS credentials (for gs:// URLs) + gcs_service_account_key: Option, ) -> Result { - let url = Url::parse(s3_path)?; + let url = Url::parse(storage_path)?; let bucket = url .host_str() - .ok_or_else(|| anyhow::anyhow!("Invalid S3 URL: missing bucket"))? + .ok_or_else(|| anyhow::anyhow!("Invalid storage URL: missing bucket"))? .to_string(); let prefix = url.path().trim_start_matches('/').to_string(); - let region = if let Some(region_str) = aws_region { - Region::new(region_str) - } else { - RegionProviderChain::default_provider() - .region() - .await - .unwrap_or_else(|| Region::new("us-east-1")) - }; + let scheme = url.scheme(); - let config = if let (Some(access_key), Some(secret_key)) = - (aws_access_key_id, aws_secret_access_key) - { - let creds = Credentials::new(access_key, secret_key, None, None, "drainage"); - aws_config::from_env() - .region(region) - .credentials_provider(creds) - .load() - .await - } else { - aws_config::from_env().region(region).load().await - }; + let store: Arc = match scheme { + "s3" => { + let mut builder = AmazonS3Builder::new().with_bucket_name(&bucket); - let client = S3Client::new(&config); + // Set region + if let Some(region) = aws_region { + builder = builder.with_region(®ion); + } + + // Set credentials if provided + if let (Some(access_key), Some(secret_key)) = + (aws_access_key_id, aws_secret_access_key) + { + builder = builder + .with_access_key_id(&access_key) + .with_secret_access_key(&secret_key); + } + + Arc::new(builder.build()?) + } + "gs" => { + let mut builder = GoogleCloudStorageBuilder::new().with_bucket_name(&bucket); + + // Set service account key if provided + if let Some(service_account_key) = gcs_service_account_key { + builder = builder.with_service_account_key(&service_account_key); + } + + Arc::new(builder.build()?) + } + _ => { + return Err(anyhow::anyhow!( + "Unsupported storage scheme: {}. Supported schemes: s3://, gs://", + scheme + )); + } + }; Ok(Self { - client, + store, bucket, prefix, }) } pub async fn list_objects(&self, prefix: &str) -> Result> { - let mut objects = Vec::new(); - let mut continuation_token: Option = None; - - loop { - let mut request = self - .client - .list_objects_v2() - .bucket(&self.bucket) - .prefix(prefix); - - if let Some(token) = continuation_token { - request = request.continuation_token(token); - } - - let response = request.send().await?; - - if let Some(contents) = response.contents { - for obj in contents { - objects.push(ObjectInfo { - key: obj.key.unwrap_or_default(), - size: obj.size, - last_modified: obj.last_modified.map(|dt| format!("{:?}", dt)), - etag: obj.e_tag, - }); - } - } + let list_prefix = if prefix.is_empty() { + None + } else { + Some(object_store::path::Path::from(prefix)) + }; - if response.is_truncated { - continuation_token = response.next_continuation_token; - } else { - break; - } + let mut objects = Vec::new(); + let mut list_stream = self.store.list(list_prefix.as_ref()); + + while let Some(meta_result) = list_stream.next().await { + let meta = meta_result?; + objects.push(ObjectInfo { + key: meta.location.to_string(), + size: meta.size as i64, + last_modified: Some(meta.last_modified.to_rfc3339()), + etag: meta.e_tag.clone(), + }); } Ok(objects) } pub async fn get_object(&self, key: &str) -> Result> { - let response = self - .client - .get_object() - .bucket(&self.bucket) - .key(key) - .send() - .await?; - - let body = response.body.collect().await?.into_bytes().to_vec(); - Ok(body) + let path = object_store::path::Path::from(key); + let get_result = self.store.get(&path).await?; + let bytes: Bytes = get_result.bytes().await?; + Ok(bytes.to_vec()) } pub fn get_bucket(&self) -> &str { @@ -124,7 +135,7 @@ pub struct ObjectInfo { #[cfg(test)] mod tests { - use super::*; + use crate::storage_client::ObjectInfo; use url::Url; #[test] @@ -171,6 +182,16 @@ mod tests { assert_eq!(url.path(), "/my-table/"); } + #[test] + fn test_gcs_url_parsing_valid() { + let gcs_path = "gs://my-bucket/my-table/"; + let url = Url::parse(gcs_path).unwrap(); + + assert_eq!(url.scheme(), "gs"); + assert_eq!(url.host_str(), Some("my-bucket")); + assert_eq!(url.path(), "/my-table/"); + } + #[test] fn test_s3_url_parsing_with_prefix() { let s3_path = "s3://my-bucket/my-table/year=2023/month=01/"; @@ -182,7 +203,17 @@ mod tests { } #[test] - fn test_s3_url_parsing_invalid() { + fn test_gcs_url_parsing_with_prefix() { + let gcs_path = "gs://my-bucket/my-table/year=2023/month=01/"; + let url = Url::parse(gcs_path).unwrap(); + + assert_eq!(url.scheme(), "gs"); + assert_eq!(url.host_str(), Some("my-bucket")); + assert_eq!(url.path(), "/my-table/year=2023/month=01/"); + } + + #[test] + fn test_url_parsing_invalid() { let invalid_path = "not-a-url"; let result = Url::parse(invalid_path); assert!(result.is_err()); @@ -201,55 +232,57 @@ mod tests { } #[test] - fn test_s3_path_components_extraction_no_trailing_slash() { - let s3_path = "s3://my-bucket/my-table"; - let url = Url::parse(s3_path).unwrap(); + fn test_gcs_path_components_extraction() { + let gcs_path = "gs://my-bucket/my-table/year=2023/month=01/"; + let url = Url::parse(gcs_path).unwrap(); let bucket = url.host_str().unwrap().to_string(); let prefix = url.path().trim_start_matches('/').to_string(); assert_eq!(bucket, "my-bucket"); - assert_eq!(prefix, "my-table"); - } - - #[test] - fn test_aws_region_creation() { - let region_str = "us-west-2"; - let region = aws_sdk_s3::config::Region::new(region_str); - - assert_eq!(region.as_ref(), "us-west-2"); + assert_eq!(prefix, "my-table/year=2023/month=01/"); } #[test] - fn test_aws_credentials_creation() { - let access_key = "AKIAIOSFODNN7EXAMPLE"; - let secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; + fn test_path_components_extraction_no_trailing_slash() { + let s3_path = "s3://my-bucket/my-table"; + let url = Url::parse(s3_path).unwrap(); - let creds = - aws_sdk_s3::config::Credentials::new(access_key, secret_key, None, None, "drainage"); + let bucket = url.host_str().unwrap().to_string(); + let prefix = url.path().trim_start_matches('/').to_string(); - assert_eq!(creds.access_key_id(), access_key); - assert_eq!(creds.secret_access_key(), secret_key); - assert_eq!(creds.session_token(), None); - assert_eq!(creds.expiry(), None); + assert_eq!(bucket, "my-bucket"); + assert_eq!(prefix, "my-table"); } #[test] - fn test_s3_path_validation() { + fn test_storage_path_validation() { let valid_paths = vec![ "s3://bucket/", "s3://bucket/path/", "s3://bucket/path/to/table/", "s3://my-bucket-name/my-table/", "s3://bucket.with.dots/table/", + "gs://bucket/", + "gs://bucket/path/", + "gs://bucket/path/to/table/", + "gs://my-bucket-name/my-table/", ]; for path in valid_paths { let result = Url::parse(path); - assert!(result.is_ok(), "Failed to parse valid S3 path: {}", path); + assert!( + result.is_ok(), + "Failed to parse valid storage path: {}", + path + ); let url = result.unwrap(); - assert_eq!(url.scheme(), "s3"); + assert!( + url.scheme() == "s3" || url.scheme() == "gs", + "Invalid scheme for path: {}", + path + ); assert!(url.host_str().is_some(), "Missing bucket in path: {}", path); } } diff --git a/src/types.rs b/src/types.rs index 1068436..083688f 100644 --- a/src/types.rs +++ b/src/types.rs @@ -348,7 +348,10 @@ impl HealthMetrics { } } - pub fn calculate_metadata_health(&mut self, metadata_files: &[crate::s3_client::ObjectInfo]) { + pub fn calculate_metadata_health( + &mut self, + metadata_files: &[crate::storage_client::ObjectInfo], + ) { self.metadata_health.metadata_file_count = metadata_files.len(); self.metadata_health.metadata_total_size_bytes = metadata_files.iter().map(|f| f.size as u64).sum(); @@ -498,7 +501,7 @@ impl HealthReport { #[cfg(test)] mod tests { - use super::*; + use crate::types::*; use std::collections::HashMap; #[test] @@ -609,6 +612,494 @@ mod tests { ); } + #[test] + fn test_health_score_calculation_with_small_files() { + let mut metrics = HealthMetrics::new(); + metrics.total_files = 100; + metrics.file_size_distribution = FileSizeDistribution { + small_files: 50, // 50% small files + medium_files: 50, + large_files: 0, + very_large_files: 0, + }; + metrics.partition_count = 10; + metrics.data_skew = DataSkewMetrics { + partition_skew_score: 0.0, + file_size_skew_score: 0.0, + largest_partition_size: 1000, + smallest_partition_size: 1000, + avg_partition_size: 1000, + partition_size_std_dev: 0.0, + }; + metrics.snapshot_health = SnapshotHealth { + snapshot_count: 5, + oldest_snapshot_age_days: 1.0, + newest_snapshot_age_days: 0.0, + avg_snapshot_age_days: 0.5, + snapshot_retention_risk: 0.0, + }; + + let score = metrics.calculate_health_score(); + // Should be penalized by 10% (50% small files * 20% penalty) + let expected_penalty = 0.5 * 0.2; + let expected_score = 1.0 - expected_penalty; + assert!( + (score - expected_score).abs() < 0.01, + "Expected score ~{}, got {}", + expected_score, + score + ); + } + + #[test] + fn test_health_score_calculation_with_very_large_files() { + let mut metrics = HealthMetrics::new(); + metrics.total_files = 100; + metrics.file_size_distribution = FileSizeDistribution { + small_files: 0, + medium_files: 90, + large_files: 0, + very_large_files: 10, // 10% very large files + }; + metrics.partition_count = 10; + metrics.data_skew = DataSkewMetrics { + partition_skew_score: 0.0, + file_size_skew_score: 0.0, + largest_partition_size: 1000, + smallest_partition_size: 1000, + avg_partition_size: 1000, + partition_size_std_dev: 0.0, + }; + metrics.snapshot_health = SnapshotHealth { + snapshot_count: 5, + oldest_snapshot_age_days: 1.0, + newest_snapshot_age_days: 0.0, + avg_snapshot_age_days: 0.5, + snapshot_retention_risk: 0.0, + }; + + let score = metrics.calculate_health_score(); + // Should be penalized by 1% (10% very large files * 10% penalty) + let expected_penalty = 0.1 * 0.1; + let expected_score = 1.0 - expected_penalty; + assert!( + (score - expected_score).abs() < 0.01, + "Expected score ~{}, got {}", + expected_score, + score + ); + } + + #[test] + fn test_health_score_calculation_with_data_skew() { + let mut metrics = HealthMetrics::new(); + metrics.total_files = 100; + metrics.file_size_distribution = FileSizeDistribution { + small_files: 0, + medium_files: 100, + large_files: 0, + very_large_files: 0, + }; + metrics.partition_count = 10; + metrics.data_skew = DataSkewMetrics { + partition_skew_score: 0.5, // 50% skew + file_size_skew_score: 0.3, // 30% skew + largest_partition_size: 2000, + smallest_partition_size: 1000, + avg_partition_size: 1500, + partition_size_std_dev: 500.0, + }; + metrics.snapshot_health = SnapshotHealth { + snapshot_count: 5, + oldest_snapshot_age_days: 1.0, + newest_snapshot_age_days: 0.0, + avg_snapshot_age_days: 0.5, + snapshot_retention_risk: 0.0, + }; + + let score = metrics.calculate_health_score(); + // Should be penalized by 10.5% (0.5 * 0.15 + 0.3 * 0.1) + let expected_penalty = 0.5 * 0.15 + 0.3 * 0.1; + let expected_score = 1.0 - expected_penalty; + assert!( + (score - expected_score).abs() < 0.01, + "Expected score ~{}, got {}", + expected_score, + score + ); + } + + #[test] + fn test_health_score_calculation_with_metadata_bloat() { + let mut metrics = HealthMetrics::new(); + metrics.total_files = 100; + metrics.file_size_distribution = FileSizeDistribution { + small_files: 0, + medium_files: 100, + large_files: 0, + very_large_files: 0, + }; + metrics.partition_count = 10; + metrics.data_skew = DataSkewMetrics { + partition_skew_score: 0.0, + file_size_skew_score: 0.0, + largest_partition_size: 1000, + smallest_partition_size: 1000, + avg_partition_size: 1000, + partition_size_std_dev: 0.0, + }; + metrics.metadata_health = MetadataHealth { + metadata_file_count: 10, + metadata_total_size_bytes: 200 * 1024 * 1024, // 200MB > 100MB threshold + avg_metadata_file_size: 20.0 * 1024.0 * 1024.0, + metadata_growth_rate: 0.0, + manifest_file_count: 0, + }; + metrics.snapshot_health = SnapshotHealth { + snapshot_count: 5, + oldest_snapshot_age_days: 1.0, + newest_snapshot_age_days: 0.0, + avg_snapshot_age_days: 0.5, + snapshot_retention_risk: 0.0, + }; + + let score = metrics.calculate_health_score(); + // Should be penalized by 5% for metadata bloat + let expected_score = 1.0 - 0.05; + assert!( + (score - expected_score).abs() < 0.01, + "Expected score ~{}, got {}", + expected_score, + score + ); + } + + #[test] + fn test_health_score_calculation_with_snapshot_retention_risk() { + let mut metrics = HealthMetrics::new(); + metrics.total_files = 100; + metrics.file_size_distribution = FileSizeDistribution { + small_files: 0, + medium_files: 100, + large_files: 0, + very_large_files: 0, + }; + metrics.partition_count = 10; + metrics.data_skew = DataSkewMetrics { + partition_skew_score: 0.0, + file_size_skew_score: 0.0, + largest_partition_size: 1000, + smallest_partition_size: 1000, + avg_partition_size: 1000, + partition_size_std_dev: 0.0, + }; + metrics.snapshot_health = SnapshotHealth { + snapshot_count: 150, // High snapshot count + oldest_snapshot_age_days: 30.0, + newest_snapshot_age_days: 0.0, + avg_snapshot_age_days: 15.0, + snapshot_retention_risk: 0.8, // High retention risk + }; + + let score = metrics.calculate_health_score(); + // Should be penalized by 8% for snapshot retention risk + let expected_score = 1.0 - 0.8 * 0.1; + assert!( + (score - expected_score).abs() < 0.01, + "Expected score ~{}, got {}", + expected_score, + score + ); + } + + #[test] + fn test_health_score_calculation_with_deletion_vector_impact() { + let mut metrics = HealthMetrics::new(); + metrics.total_files = 100; + metrics.file_size_distribution = FileSizeDistribution { + small_files: 0, + medium_files: 100, + large_files: 0, + very_large_files: 0, + }; + metrics.partition_count = 10; + metrics.data_skew = DataSkewMetrics { + partition_skew_score: 0.0, + file_size_skew_score: 0.0, + largest_partition_size: 1000, + smallest_partition_size: 1000, + avg_partition_size: 1000, + partition_size_std_dev: 0.0, + }; + metrics.snapshot_health = SnapshotHealth { + snapshot_count: 5, + oldest_snapshot_age_days: 1.0, + newest_snapshot_age_days: 0.0, + avg_snapshot_age_days: 0.5, + snapshot_retention_risk: 0.0, + }; + metrics.deletion_vector_metrics = Some(DeletionVectorMetrics { + deletion_vector_count: 10, + total_deletion_vector_size_bytes: 1024 * 1024, + avg_deletion_vector_size_bytes: 102.4 * 1024.0, + deletion_vector_age_days: 5.0, + deleted_rows_count: 1000, + deletion_vector_impact_score: 0.6, // High impact + }); + + let score = metrics.calculate_health_score(); + // Should be penalized by 9% for deletion vector impact + let expected_score = 1.0 - 0.6 * 0.15; + assert!( + (score - expected_score).abs() < 0.01, + "Expected score ~{}, got {}", + expected_score, + score + ); + } + + #[test] + fn test_health_score_calculation_with_schema_instability() { + let mut metrics = HealthMetrics::new(); + metrics.total_files = 100; + metrics.file_size_distribution = FileSizeDistribution { + small_files: 0, + medium_files: 100, + large_files: 0, + very_large_files: 0, + }; + metrics.partition_count = 10; + metrics.data_skew = DataSkewMetrics { + partition_skew_score: 0.0, + file_size_skew_score: 0.0, + largest_partition_size: 1000, + smallest_partition_size: 1000, + avg_partition_size: 1000, + partition_size_std_dev: 0.0, + }; + metrics.snapshot_health = SnapshotHealth { + snapshot_count: 5, + oldest_snapshot_age_days: 1.0, + newest_snapshot_age_days: 0.0, + avg_snapshot_age_days: 0.5, + snapshot_retention_risk: 0.0, + }; + metrics.schema_evolution = Some(SchemaEvolutionMetrics { + total_schema_changes: 20, + breaking_changes: 5, + non_breaking_changes: 15, + schema_stability_score: 0.3, // Low stability + days_since_last_change: 1.0, + schema_change_frequency: 0.1, + current_schema_version: 20, + }); + + let score = metrics.calculate_health_score(); + // Should be penalized by 14% for schema instability (1.0 - 0.3) * 0.2 + let expected_score = 1.0 - (1.0 - 0.3) * 0.2; + assert!( + (score - expected_score).abs() < 0.01, + "Expected score ~{}, got {}", + expected_score, + score + ); + } + + #[test] + fn test_health_score_calculation_with_time_travel_costs() { + let mut metrics = HealthMetrics::new(); + metrics.total_files = 100; + metrics.file_size_distribution = FileSizeDistribution { + small_files: 0, + medium_files: 100, + large_files: 0, + very_large_files: 0, + }; + metrics.partition_count = 10; + metrics.data_skew = DataSkewMetrics { + partition_skew_score: 0.0, + file_size_skew_score: 0.0, + largest_partition_size: 1000, + smallest_partition_size: 1000, + avg_partition_size: 1000, + partition_size_std_dev: 0.0, + }; + metrics.snapshot_health = SnapshotHealth { + snapshot_count: 5, + oldest_snapshot_age_days: 1.0, + newest_snapshot_age_days: 0.0, + avg_snapshot_age_days: 0.5, + snapshot_retention_risk: 0.0, + }; + metrics.time_travel_metrics = Some(TimeTravelMetrics { + total_snapshots: 100, + oldest_snapshot_age_days: 30.0, + newest_snapshot_age_days: 0.0, + total_historical_size_bytes: 10 * 1024 * 1024 * 1024, // 10GB + avg_snapshot_size_bytes: 100.0 * 1024.0 * 1024.0, + storage_cost_impact_score: 0.7, // High cost impact + retention_efficiency_score: 0.4, // Low efficiency + recommended_retention_days: 7, + }); + + let score = metrics.calculate_health_score(); + // Should be penalized by 10.5% (0.7 * 0.1 + (1.0 - 0.4) * 0.05) + let expected_score = 1.0 - (0.7 * 0.1 + (1.0 - 0.4) * 0.05); + assert!( + (score - expected_score).abs() < 0.01, + "Expected score ~{}, got {}", + expected_score, + score + ); + } + + #[test] + fn test_health_score_calculation_with_data_quality_issues() { + let mut metrics = HealthMetrics::new(); + metrics.total_files = 100; + metrics.file_size_distribution = FileSizeDistribution { + small_files: 0, + medium_files: 100, + large_files: 0, + very_large_files: 0, + }; + metrics.partition_count = 10; + metrics.data_skew = DataSkewMetrics { + partition_skew_score: 0.0, + file_size_skew_score: 0.0, + largest_partition_size: 1000, + smallest_partition_size: 1000, + avg_partition_size: 1000, + partition_size_std_dev: 0.0, + }; + metrics.snapshot_health = SnapshotHealth { + snapshot_count: 5, + oldest_snapshot_age_days: 1.0, + newest_snapshot_age_days: 0.0, + avg_snapshot_age_days: 0.5, + snapshot_retention_risk: 0.0, + }; + metrics.table_constraints = Some(TableConstraintsMetrics { + total_constraints: 5, + check_constraints: 2, + not_null_constraints: 2, + unique_constraints: 1, + foreign_key_constraints: 0, + constraint_violation_risk: 0.8, // High violation risk + data_quality_score: 0.2, // Poor data quality + constraint_coverage_score: 0.3, // Low coverage + }); + + let score = metrics.calculate_health_score(); + // Should be penalized by 22% ((1.0 - 0.2) * 0.15 + 0.8 * 0.1) + let expected_score = 1.0 - ((1.0 - 0.2) * 0.15 + 0.8 * 0.1); + assert!( + (score - expected_score).abs() < 0.01, + "Expected score ~{}, got {}", + expected_score, + score + ); + } + + #[test] + fn test_health_score_calculation_with_compaction_opportunities() { + let mut metrics = HealthMetrics::new(); + metrics.total_files = 100; + metrics.file_size_distribution = FileSizeDistribution { + small_files: 0, + medium_files: 100, + large_files: 0, + very_large_files: 0, + }; + metrics.partition_count = 10; + metrics.data_skew = DataSkewMetrics { + partition_skew_score: 0.0, + file_size_skew_score: 0.0, + largest_partition_size: 1000, + smallest_partition_size: 1000, + avg_partition_size: 1000, + partition_size_std_dev: 0.0, + }; + metrics.snapshot_health = SnapshotHealth { + snapshot_count: 5, + oldest_snapshot_age_days: 1.0, + newest_snapshot_age_days: 0.0, + avg_snapshot_age_days: 0.5, + snapshot_retention_risk: 0.0, + }; + metrics.file_compaction = Some(FileCompactionMetrics { + compaction_opportunity_score: 0.9, // High opportunity + small_files_count: 50, + small_files_size_bytes: 100 * 1024 * 1024, + potential_compaction_files: 50, + estimated_compaction_savings_bytes: 20 * 1024 * 1024, + recommended_target_file_size_bytes: 128 * 1024 * 1024, + compaction_priority: "high".to_string(), + z_order_opportunity: true, + z_order_columns: vec!["col1".to_string(), "col2".to_string()], + }); + + let score = metrics.calculate_health_score(); + // Should be penalized by 1% for compaction opportunities (1.0 - 0.9) * 0.1 + let expected_score = 1.0 - (1.0 - 0.9) * 0.1; + assert!( + (score - expected_score).abs() < 0.01, + "Expected score ~{}, got {}", + expected_score, + score + ); + } + + #[test] + fn test_health_score_calculation_minimum_score() { + let mut metrics = HealthMetrics::new(); + metrics.total_files = 100; + metrics.unreferenced_files = vec![ + FileInfo { + path: "unreferenced.parquet".to_string(), + size_bytes: 1000, + last_modified: None, + is_referenced: false, + }; + 100 + ]; // All files unreferenced + metrics.file_size_distribution = FileSizeDistribution { + small_files: 100, // All small files + medium_files: 0, + large_files: 0, + very_large_files: 0, + }; + metrics.partition_count = 1; + metrics.data_skew = DataSkewMetrics { + partition_skew_score: 1.0, // Maximum skew + file_size_skew_score: 1.0, // Maximum skew + largest_partition_size: 1000, + smallest_partition_size: 1000, + avg_partition_size: 1000, + partition_size_std_dev: 0.0, + }; + metrics.snapshot_health = SnapshotHealth { + snapshot_count: 1000, + oldest_snapshot_age_days: 365.0, + newest_snapshot_age_days: 0.0, + avg_snapshot_age_days: 182.5, + snapshot_retention_risk: 1.0, // Maximum risk + }; + + let score = metrics.calculate_health_score(); + // Should be close to 0 but not negative + assert!( + score >= 0.0, + "Health score should not be negative, got {}", + score + ); + assert!( + score < 0.2, + "Health score should be very low, got {}", + score + ); + } + #[test] fn test_calculate_data_skew_empty_partitions() { let mut metrics = HealthMetrics::new(); @@ -658,17 +1149,46 @@ mod tests { assert_eq!(metrics.data_skew.avg_partition_size, 1000); } + #[test] + fn test_calculate_data_skew_high_skew() { + let mut metrics = HealthMetrics::new(); + metrics.partitions = vec![ + PartitionInfo { + partition_values: HashMap::new(), + file_count: 1, + total_size_bytes: 100, + avg_file_size_bytes: 100.0, + files: vec![], + }, + PartitionInfo { + partition_values: HashMap::new(), + file_count: 99, + total_size_bytes: 9900, + avg_file_size_bytes: 100.0, + files: vec![], + }, + ]; + + metrics.calculate_data_skew(); + + // High skew should result in high skew scores + assert!(metrics.data_skew.partition_skew_score > 0.5); + assert!(metrics.data_skew.file_size_skew_score > 0.5); + assert_eq!(metrics.data_skew.largest_partition_size, 9900); + assert_eq!(metrics.data_skew.smallest_partition_size, 100); + } + #[test] fn test_calculate_metadata_health() { let mut metrics = HealthMetrics::new(); let metadata_files = vec![ - crate::s3_client::ObjectInfo { + crate::storage_client::ObjectInfo { key: "metadata1.json".to_string(), size: 1000, last_modified: Some("2023-01-01T00:00:00Z".to_string()), etag: Some("etag1".to_string()), }, - crate::s3_client::ObjectInfo { + crate::storage_client::ObjectInfo { key: "metadata2.json".to_string(), size: 2000, last_modified: Some("2023-01-02T00:00:00Z".to_string()), @@ -693,6 +1213,36 @@ mod tests { assert_eq!(metrics.snapshot_health.snapshot_retention_risk, 0.0); } + #[test] + fn test_calculate_snapshot_health_medium_risk() { + let mut metrics = HealthMetrics::new(); + + metrics.calculate_snapshot_health(30); + + assert_eq!(metrics.snapshot_health.snapshot_count, 30); + assert_eq!(metrics.snapshot_health.snapshot_retention_risk, 0.2); + } + + #[test] + fn test_calculate_snapshot_health_high_risk() { + let mut metrics = HealthMetrics::new(); + + metrics.calculate_snapshot_health(75); + + assert_eq!(metrics.snapshot_health.snapshot_count, 75); + assert_eq!(metrics.snapshot_health.snapshot_retention_risk, 0.5); + } + + #[test] + fn test_calculate_snapshot_health_critical_risk() { + let mut metrics = HealthMetrics::new(); + + metrics.calculate_snapshot_health(150); + + assert_eq!(metrics.snapshot_health.snapshot_count, 150); + assert_eq!(metrics.snapshot_health.snapshot_retention_risk, 0.8); + } + #[test] fn test_health_report_new() { let report = HealthReport::new("s3://bucket/table".to_string(), "delta".to_string()); @@ -703,4 +1253,193 @@ mod tests { assert_eq!(report.health_score, 0.0); assert_eq!(report.metrics.total_files, 0); } + + #[test] + fn test_file_info_creation() { + let file_info = FileInfo { + path: "test.parquet".to_string(), + size_bytes: 1024, + last_modified: Some("2023-01-01T00:00:00Z".to_string()), + is_referenced: true, + }; + + assert_eq!(file_info.path, "test.parquet"); + assert_eq!(file_info.size_bytes, 1024); + assert_eq!( + file_info.last_modified, + Some("2023-01-01T00:00:00Z".to_string()) + ); + assert!(file_info.is_referenced); + } + + #[test] + fn test_partition_info_creation() { + let mut partition_values = HashMap::new(); + partition_values.insert("year".to_string(), "2023".to_string()); + partition_values.insert("month".to_string(), "01".to_string()); + + let partition_info = PartitionInfo { + partition_values: partition_values.clone(), + file_count: 10, + total_size_bytes: 10000, + avg_file_size_bytes: 1000.0, + files: vec![], + }; + + assert_eq!(partition_info.partition_values, partition_values); + assert_eq!(partition_info.file_count, 10); + assert_eq!(partition_info.total_size_bytes, 10000); + assert_eq!(partition_info.avg_file_size_bytes, 1000.0); + assert_eq!(partition_info.files.len(), 0); + } + + #[test] + fn test_clustering_info_creation() { + let clustering_info = ClusteringInfo { + clustering_columns: vec!["col1".to_string(), "col2".to_string()], + cluster_count: 5, + avg_files_per_cluster: 20.0, + avg_cluster_size_bytes: 2000.0, + }; + + assert_eq!(clustering_info.clustering_columns, vec!["col1", "col2"]); + assert_eq!(clustering_info.cluster_count, 5); + assert_eq!(clustering_info.avg_files_per_cluster, 20.0); + assert_eq!(clustering_info.avg_cluster_size_bytes, 2000.0); + } + + #[test] + fn test_file_size_distribution_creation() { + let distribution = FileSizeDistribution { + small_files: 10, + medium_files: 20, + large_files: 5, + very_large_files: 1, + }; + + assert_eq!(distribution.small_files, 10); + assert_eq!(distribution.medium_files, 20); + assert_eq!(distribution.large_files, 5); + assert_eq!(distribution.very_large_files, 1); + } + + #[test] + fn test_deletion_vector_metrics_creation() { + let dv_metrics = DeletionVectorMetrics { + deletion_vector_count: 5, + total_deletion_vector_size_bytes: 1024 * 1024, + avg_deletion_vector_size_bytes: 204.8 * 1024.0, + deletion_vector_age_days: 10.0, + deleted_rows_count: 1000, + deletion_vector_impact_score: 0.5, + }; + + assert_eq!(dv_metrics.deletion_vector_count, 5); + assert_eq!(dv_metrics.total_deletion_vector_size_bytes, 1024 * 1024); + assert_eq!(dv_metrics.avg_deletion_vector_size_bytes, 204.8 * 1024.0); + assert_eq!(dv_metrics.deletion_vector_age_days, 10.0); + assert_eq!(dv_metrics.deleted_rows_count, 1000); + assert_eq!(dv_metrics.deletion_vector_impact_score, 0.5); + } + + #[test] + fn test_schema_evolution_metrics_creation() { + let schema_metrics = SchemaEvolutionMetrics { + total_schema_changes: 10, + breaking_changes: 2, + non_breaking_changes: 8, + schema_stability_score: 0.8, + days_since_last_change: 5.0, + schema_change_frequency: 0.1, + current_schema_version: 10, + }; + + assert_eq!(schema_metrics.total_schema_changes, 10); + assert_eq!(schema_metrics.breaking_changes, 2); + assert_eq!(schema_metrics.non_breaking_changes, 8); + assert_eq!(schema_metrics.schema_stability_score, 0.8); + assert_eq!(schema_metrics.days_since_last_change, 5.0); + assert_eq!(schema_metrics.schema_change_frequency, 0.1); + assert_eq!(schema_metrics.current_schema_version, 10); + } + + #[test] + fn test_time_travel_metrics_creation() { + let tt_metrics = TimeTravelMetrics { + total_snapshots: 50, + oldest_snapshot_age_days: 30.0, + newest_snapshot_age_days: 0.0, + total_historical_size_bytes: 5 * 1024 * 1024 * 1024, + avg_snapshot_size_bytes: 100.0 * 1024.0 * 1024.0, + storage_cost_impact_score: 0.3, + retention_efficiency_score: 0.7, + recommended_retention_days: 14, + }; + + assert_eq!(tt_metrics.total_snapshots, 50); + assert_eq!(tt_metrics.oldest_snapshot_age_days, 30.0); + assert_eq!(tt_metrics.newest_snapshot_age_days, 0.0); + assert_eq!( + tt_metrics.total_historical_size_bytes, + 5 * 1024 * 1024 * 1024 + ); + assert_eq!(tt_metrics.avg_snapshot_size_bytes, 100.0 * 1024.0 * 1024.0); + assert_eq!(tt_metrics.storage_cost_impact_score, 0.3); + assert_eq!(tt_metrics.retention_efficiency_score, 0.7); + assert_eq!(tt_metrics.recommended_retention_days, 14); + } + + #[test] + fn test_table_constraints_metrics_creation() { + let constraint_metrics = TableConstraintsMetrics { + total_constraints: 8, + check_constraints: 3, + not_null_constraints: 4, + unique_constraints: 1, + foreign_key_constraints: 0, + constraint_violation_risk: 0.2, + data_quality_score: 0.9, + constraint_coverage_score: 0.8, + }; + + assert_eq!(constraint_metrics.total_constraints, 8); + assert_eq!(constraint_metrics.check_constraints, 3); + assert_eq!(constraint_metrics.not_null_constraints, 4); + assert_eq!(constraint_metrics.unique_constraints, 1); + assert_eq!(constraint_metrics.foreign_key_constraints, 0); + assert_eq!(constraint_metrics.constraint_violation_risk, 0.2); + assert_eq!(constraint_metrics.data_quality_score, 0.9); + assert_eq!(constraint_metrics.constraint_coverage_score, 0.8); + } + + #[test] + fn test_file_compaction_metrics_creation() { + let compaction_metrics = FileCompactionMetrics { + compaction_opportunity_score: 0.7, + small_files_count: 25, + small_files_size_bytes: 50 * 1024 * 1024, + potential_compaction_files: 25, + estimated_compaction_savings_bytes: 10 * 1024 * 1024, + recommended_target_file_size_bytes: 128 * 1024 * 1024, + compaction_priority: "medium".to_string(), + z_order_opportunity: true, + z_order_columns: vec!["col1".to_string(), "col2".to_string()], + }; + + assert_eq!(compaction_metrics.compaction_opportunity_score, 0.7); + assert_eq!(compaction_metrics.small_files_count, 25); + assert_eq!(compaction_metrics.small_files_size_bytes, 50 * 1024 * 1024); + assert_eq!(compaction_metrics.potential_compaction_files, 25); + assert_eq!( + compaction_metrics.estimated_compaction_savings_bytes, + 10 * 1024 * 1024 + ); + assert_eq!( + compaction_metrics.recommended_target_file_size_bytes, + 128 * 1024 * 1024 + ); + assert_eq!(compaction_metrics.compaction_priority, "medium"); + assert!(compaction_metrics.z_order_opportunity); + assert_eq!(compaction_metrics.z_order_columns, vec!["col1", "col2"]); + } } diff --git a/src/types_tests.rs b/src/types_tests.rs deleted file mode 100644 index e78c11d..0000000 --- a/src/types_tests.rs +++ /dev/null @@ -1,863 +0,0 @@ -#[cfg(test)] -mod tests { - use super::*; - use crate::types::*; - use std::collections::HashMap; - - #[test] - fn test_health_metrics_new() { - let metrics = HealthMetrics::new(); - - assert_eq!(metrics.total_files, 0); - assert_eq!(metrics.total_size_bytes, 0); - assert_eq!(metrics.unreferenced_files.len(), 0); - assert_eq!(metrics.unreferenced_size_bytes, 0); - assert_eq!(metrics.partition_count, 0); - assert_eq!(metrics.partitions.len(), 0); - assert!(metrics.clustering.is_none()); - assert_eq!(metrics.avg_file_size_bytes, 0.0); - assert_eq!(metrics.file_size_distribution.small_files, 0); - assert_eq!(metrics.file_size_distribution.medium_files, 0); - assert_eq!(metrics.file_size_distribution.large_files, 0); - assert_eq!(metrics.file_size_distribution.very_large_files, 0); - assert_eq!(metrics.recommendations.len(), 0); - assert_eq!(metrics.health_score, 0.0); - } - - #[test] - fn test_health_score_calculation_perfect_health() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 0, - medium_files: 100, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.0, - file_size_skew_score: 0.0, - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 5, - oldest_snapshot_age_days: 1.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 0.5, - snapshot_retention_risk: 0.0, - }; - - let score = metrics.calculate_health_score(); - assert!((score - 1.0).abs() < 0.01, "Expected perfect health score, got {}", score); - } - - #[test] - fn test_health_score_calculation_with_unreferenced_files() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.unreferenced_files = vec![ - FileInfo { - path: "unreferenced1.parquet".to_string(), - size_bytes: 1000, - last_modified: None, - is_referenced: false, - }, - FileInfo { - path: "unreferenced2.parquet".to_string(), - size_bytes: 2000, - last_modified: None, - is_referenced: false, - }, - ]; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 0, - medium_files: 100, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.0, - file_size_skew_score: 0.0, - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 5, - oldest_snapshot_age_days: 1.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 0.5, - snapshot_retention_risk: 0.0, - }; - - let score = metrics.calculate_health_score(); - // Should be penalized by 2% (2 unreferenced files out of 100 total) - let expected_penalty = 0.02 * 0.3; // 2% * 30% penalty - let expected_score = 1.0 - expected_penalty; - assert!((score - expected_score).abs() < 0.01, "Expected score ~{}, got {}", expected_score, score); - } - - #[test] - fn test_health_score_calculation_with_small_files() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 50, // 50% small files - medium_files: 50, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.0, - file_size_skew_score: 0.0, - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 5, - oldest_snapshot_age_days: 1.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 0.5, - snapshot_retention_risk: 0.0, - }; - - let score = metrics.calculate_health_score(); - // Should be penalized by 10% (50% small files * 20% penalty) - let expected_penalty = 0.5 * 0.2; - let expected_score = 1.0 - expected_penalty; - assert!((score - expected_score).abs() < 0.01, "Expected score ~{}, got {}", expected_score, score); - } - - #[test] - fn test_health_score_calculation_with_very_large_files() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 0, - medium_files: 90, - large_files: 0, - very_large_files: 10, // 10% very large files - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.0, - file_size_skew_score: 0.0, - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 5, - oldest_snapshot_age_days: 1.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 0.5, - snapshot_retention_risk: 0.0, - }; - - let score = metrics.calculate_health_score(); - // Should be penalized by 1% (10% very large files * 10% penalty) - let expected_penalty = 0.1 * 0.1; - let expected_score = 1.0 - expected_penalty; - assert!((score - expected_score).abs() < 0.01, "Expected score ~{}, got {}", expected_score, score); - } - - #[test] - fn test_health_score_calculation_with_data_skew() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 0, - medium_files: 100, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.5, // 50% skew - file_size_skew_score: 0.3, // 30% skew - largest_partition_size: 2000, - smallest_partition_size: 1000, - avg_partition_size: 1500, - partition_size_std_dev: 500.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 5, - oldest_snapshot_age_days: 1.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 0.5, - snapshot_retention_risk: 0.0, - }; - - let score = metrics.calculate_health_score(); - // Should be penalized by 10.5% (0.5 * 0.15 + 0.3 * 0.1) - let expected_penalty = 0.5 * 0.15 + 0.3 * 0.1; - let expected_score = 1.0 - expected_penalty; - assert!((score - expected_score).abs() < 0.01, "Expected score ~{}, got {}", expected_score, score); - } - - #[test] - fn test_health_score_calculation_with_metadata_bloat() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 0, - medium_files: 100, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.0, - file_size_skew_score: 0.0, - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.metadata_health = MetadataHealth { - metadata_file_count: 10, - metadata_total_size_bytes: 200 * 1024 * 1024, // 200MB > 100MB threshold - avg_metadata_file_size: 20.0 * 1024.0 * 1024.0, - metadata_growth_rate: 0.0, - manifest_file_count: 0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 5, - oldest_snapshot_age_days: 1.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 0.5, - snapshot_retention_risk: 0.0, - }; - - let score = metrics.calculate_health_score(); - // Should be penalized by 5% for metadata bloat - let expected_score = 1.0 - 0.05; - assert!((score - expected_score).abs() < 0.01, "Expected score ~{}, got {}", expected_score, score); - } - - #[test] - fn test_health_score_calculation_with_snapshot_retention_risk() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 0, - medium_files: 100, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.0, - file_size_skew_score: 0.0, - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 150, // High snapshot count - oldest_snapshot_age_days: 30.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 15.0, - snapshot_retention_risk: 0.8, // High retention risk - }; - - let score = metrics.calculate_health_score(); - // Should be penalized by 8% for snapshot retention risk - let expected_score = 1.0 - 0.8 * 0.1; - assert!((score - expected_score).abs() < 0.01, "Expected score ~{}, got {}", expected_score, score); - } - - #[test] - fn test_health_score_calculation_with_deletion_vector_impact() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 0, - medium_files: 100, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.0, - file_size_skew_score: 0.0, - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 5, - oldest_snapshot_age_days: 1.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 0.5, - snapshot_retention_risk: 0.0, - }; - metrics.deletion_vector_metrics = Some(DeletionVectorMetrics { - deletion_vector_count: 10, - total_deletion_vector_size_bytes: 1024 * 1024, - avg_deletion_vector_size_bytes: 102.4 * 1024.0, - deletion_vector_age_days: 5.0, - deleted_rows_count: 1000, - deletion_vector_impact_score: 0.6, // High impact - }); - - let score = metrics.calculate_health_score(); - // Should be penalized by 9% for deletion vector impact - let expected_score = 1.0 - 0.6 * 0.15; - assert!((score - expected_score).abs() < 0.01, "Expected score ~{}, got {}", expected_score, score); - } - - #[test] - fn test_health_score_calculation_with_schema_instability() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 0, - medium_files: 100, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.0, - file_size_skew_score: 0.0, - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 5, - oldest_snapshot_age_days: 1.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 0.5, - snapshot_retention_risk: 0.0, - }; - metrics.schema_evolution = Some(SchemaEvolutionMetrics { - total_schema_changes: 20, - breaking_changes: 5, - non_breaking_changes: 15, - schema_stability_score: 0.3, // Low stability - days_since_last_change: 1.0, - schema_change_frequency: 0.1, - current_schema_version: 20, - }); - - let score = metrics.calculate_health_score(); - // Should be penalized by 14% for schema instability (1.0 - 0.3) * 0.2 - let expected_score = 1.0 - (1.0 - 0.3) * 0.2; - assert!((score - expected_score).abs() < 0.01, "Expected score ~{}, got {}", expected_score, score); - } - - #[test] - fn test_health_score_calculation_with_time_travel_costs() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 0, - medium_files: 100, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.0, - file_size_skew_score: 0.0, - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 5, - oldest_snapshot_age_days: 1.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 0.5, - snapshot_retention_risk: 0.0, - }; - metrics.time_travel_metrics = Some(TimeTravelMetrics { - total_snapshots: 100, - oldest_snapshot_age_days: 30.0, - newest_snapshot_age_days: 0.0, - total_historical_size_bytes: 10 * 1024 * 1024 * 1024, // 10GB - avg_snapshot_size_bytes: 100.0 * 1024.0 * 1024.0, - storage_cost_impact_score: 0.7, // High cost impact - retention_efficiency_score: 0.4, // Low efficiency - recommended_retention_days: 7, - }); - - let score = metrics.calculate_health_score(); - // Should be penalized by 10.5% (0.7 * 0.1 + (1.0 - 0.4) * 0.05) - let expected_score = 1.0 - (0.7 * 0.1 + (1.0 - 0.4) * 0.05); - assert!((score - expected_score).abs() < 0.01, "Expected score ~{}, got {}", expected_score, score); - } - - #[test] - fn test_health_score_calculation_with_data_quality_issues() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 0, - medium_files: 100, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.0, - file_size_skew_score: 0.0, - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 5, - oldest_snapshot_age_days: 1.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 0.5, - snapshot_retention_risk: 0.0, - }; - metrics.table_constraints = Some(TableConstraintsMetrics { - total_constraints: 5, - check_constraints: 2, - not_null_constraints: 2, - unique_constraints: 1, - foreign_key_constraints: 0, - constraint_violation_risk: 0.8, // High violation risk - data_quality_score: 0.2, // Poor data quality - constraint_coverage_score: 0.3, // Low coverage - }); - - let score = metrics.calculate_health_score(); - // Should be penalized by 22% ((1.0 - 0.2) * 0.15 + 0.8 * 0.1) - let expected_score = 1.0 - ((1.0 - 0.2) * 0.15 + 0.8 * 0.1); - assert!((score - expected_score).abs() < 0.01, "Expected score ~{}, got {}", expected_score, score); - } - - #[test] - fn test_health_score_calculation_with_compaction_opportunities() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.file_size_distribution = FileSizeDistribution { - small_files: 0, - medium_files: 100, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 10; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 0.0, - file_size_skew_score: 0.0, - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 5, - oldest_snapshot_age_days: 1.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 0.5, - snapshot_retention_risk: 0.0, - }; - metrics.file_compaction = Some(FileCompactionMetrics { - compaction_opportunity_score: 0.9, // High opportunity - small_files_count: 50, - small_files_size_bytes: 100 * 1024 * 1024, - potential_compaction_files: 50, - estimated_compaction_savings_bytes: 20 * 1024 * 1024, - recommended_target_file_size_bytes: 128 * 1024 * 1024, - compaction_priority: "high".to_string(), - z_order_opportunity: true, - z_order_columns: vec!["col1".to_string(), "col2".to_string()], - }); - - let score = metrics.calculate_health_score(); - // Should be penalized by 1% for compaction opportunities (1.0 - 0.9) * 0.1 - let expected_score = 1.0 - (1.0 - 0.9) * 0.1; - assert!((score - expected_score).abs() < 0.01, "Expected score ~{}, got {}", expected_score, score); - } - - #[test] - fn test_health_score_calculation_minimum_score() { - let mut metrics = HealthMetrics::new(); - metrics.total_files = 100; - metrics.unreferenced_files = vec![FileInfo { - path: "unreferenced.parquet".to_string(), - size_bytes: 1000, - last_modified: None, - is_referenced: false, - }; 100]; // All files unreferenced - metrics.file_size_distribution = FileSizeDistribution { - small_files: 100, // All small files - medium_files: 0, - large_files: 0, - very_large_files: 0, - }; - metrics.partition_count = 1; - metrics.data_skew = DataSkewMetrics { - partition_skew_score: 1.0, // Maximum skew - file_size_skew_score: 1.0, // Maximum skew - largest_partition_size: 1000, - smallest_partition_size: 1000, - avg_partition_size: 1000, - partition_size_std_dev: 0.0, - }; - metrics.snapshot_health = SnapshotHealth { - snapshot_count: 1000, - oldest_snapshot_age_days: 365.0, - newest_snapshot_age_days: 0.0, - avg_snapshot_age_days: 182.5, - snapshot_retention_risk: 1.0, // Maximum risk - }; - - let score = metrics.calculate_health_score(); - // Should be close to 0 but not negative - assert!(score >= 0.0, "Health score should not be negative, got {}", score); - assert!(score < 0.1, "Health score should be very low, got {}", score); - } - - #[test] - fn test_calculate_data_skew_empty_partitions() { - let mut metrics = HealthMetrics::new(); - metrics.partitions = vec![]; - - metrics.calculate_data_skew(); - - // Should not crash and should keep default values - assert_eq!(metrics.data_skew.partition_skew_score, 0.0); - assert_eq!(metrics.data_skew.file_size_skew_score, 0.0); - } - - #[test] - fn test_calculate_data_skew_perfect_distribution() { - let mut metrics = HealthMetrics::new(); - metrics.partitions = vec![ - PartitionInfo { - partition_values: HashMap::new(), - file_count: 10, - total_size_bytes: 1000, - avg_file_size_bytes: 100.0, - files: vec![], - }, - PartitionInfo { - partition_values: HashMap::new(), - file_count: 10, - total_size_bytes: 1000, - avg_file_size_bytes: 100.0, - files: vec![], - }, - PartitionInfo { - partition_values: HashMap::new(), - file_count: 10, - total_size_bytes: 1000, - avg_file_size_bytes: 100.0, - files: vec![], - }, - ]; - - metrics.calculate_data_skew(); - - // Perfect distribution should have 0 skew - assert_eq!(metrics.data_skew.partition_skew_score, 0.0); - assert_eq!(metrics.data_skew.file_size_skew_score, 0.0); - assert_eq!(metrics.data_skew.largest_partition_size, 1000); - assert_eq!(metrics.data_skew.smallest_partition_size, 1000); - assert_eq!(metrics.data_skew.avg_partition_size, 1000); - } - - #[test] - fn test_calculate_data_skew_high_skew() { - let mut metrics = HealthMetrics::new(); - metrics.partitions = vec![ - PartitionInfo { - partition_values: HashMap::new(), - file_count: 1, - total_size_bytes: 100, - avg_file_size_bytes: 100.0, - files: vec![], - }, - PartitionInfo { - partition_values: HashMap::new(), - file_count: 99, - total_size_bytes: 9900, - avg_file_size_bytes: 100.0, - files: vec![], - }, - ]; - - metrics.calculate_data_skew(); - - // High skew should result in high skew scores - assert!(metrics.data_skew.partition_skew_score > 0.5); - assert!(metrics.data_skew.file_size_skew_score > 0.5); - assert_eq!(metrics.data_skew.largest_partition_size, 9900); - assert_eq!(metrics.data_skew.smallest_partition_size, 100); - } - - #[test] - fn test_calculate_metadata_health() { - let mut metrics = HealthMetrics::new(); - let metadata_files = vec![ - crate::s3_client::ObjectInfo { - key: "metadata1.json".to_string(), - size: 1000, - last_modified: Some("2023-01-01T00:00:00Z".to_string()), - etag: Some("etag1".to_string()), - }, - crate::s3_client::ObjectInfo { - key: "metadata2.json".to_string(), - size: 2000, - last_modified: Some("2023-01-02T00:00:00Z".to_string()), - etag: Some("etag2".to_string()), - }, - ]; - - metrics.calculate_metadata_health(&metadata_files); - - assert_eq!(metrics.metadata_health.metadata_file_count, 2); - assert_eq!(metrics.metadata_health.metadata_total_size_bytes, 3000); - assert_eq!(metrics.metadata_health.avg_metadata_file_size, 1500.0); - } - - #[test] - fn test_calculate_snapshot_health_low_risk() { - let mut metrics = HealthMetrics::new(); - - metrics.calculate_snapshot_health(5); - - assert_eq!(metrics.snapshot_health.snapshot_count, 5); - assert_eq!(metrics.snapshot_health.snapshot_retention_risk, 0.0); - } - - #[test] - fn test_calculate_snapshot_health_medium_risk() { - let mut metrics = HealthMetrics::new(); - - metrics.calculate_snapshot_health(30); - - assert_eq!(metrics.snapshot_health.snapshot_count, 30); - assert_eq!(metrics.snapshot_health.snapshot_retention_risk, 0.2); - } - - #[test] - fn test_calculate_snapshot_health_high_risk() { - let mut metrics = HealthMetrics::new(); - - metrics.calculate_snapshot_health(75); - - assert_eq!(metrics.snapshot_health.snapshot_count, 75); - assert_eq!(metrics.snapshot_health.snapshot_retention_risk, 0.5); - } - - #[test] - fn test_calculate_snapshot_health_critical_risk() { - let mut metrics = HealthMetrics::new(); - - metrics.calculate_snapshot_health(150); - - assert_eq!(metrics.snapshot_health.snapshot_count, 150); - assert_eq!(metrics.snapshot_health.snapshot_retention_risk, 0.8); - } - - #[test] - fn test_health_report_new() { - let report = HealthReport::new("s3://bucket/table".to_string(), "delta".to_string()); - - assert_eq!(report.table_path, "s3://bucket/table"); - assert_eq!(report.table_type, "delta"); - assert!(!report.analysis_timestamp.is_empty()); - assert_eq!(report.health_score, 0.0); - assert_eq!(report.metrics.total_files, 0); - } - - #[test] - fn test_file_info_creation() { - let file_info = FileInfo { - path: "test.parquet".to_string(), - size_bytes: 1024, - last_modified: Some("2023-01-01T00:00:00Z".to_string()), - is_referenced: true, - }; - - assert_eq!(file_info.path, "test.parquet"); - assert_eq!(file_info.size_bytes, 1024); - assert_eq!(file_info.last_modified, Some("2023-01-01T00:00:00Z".to_string())); - assert!(file_info.is_referenced); - } - - #[test] - fn test_partition_info_creation() { - let mut partition_values = HashMap::new(); - partition_values.insert("year".to_string(), "2023".to_string()); - partition_values.insert("month".to_string(), "01".to_string()); - - let partition_info = PartitionInfo { - partition_values: partition_values.clone(), - file_count: 10, - total_size_bytes: 10000, - avg_file_size_bytes: 1000.0, - files: vec![], - }; - - assert_eq!(partition_info.partition_values, partition_values); - assert_eq!(partition_info.file_count, 10); - assert_eq!(partition_info.total_size_bytes, 10000); - assert_eq!(partition_info.avg_file_size_bytes, 1000.0); - assert_eq!(partition_info.files.len(), 0); - } - - #[test] - fn test_clustering_info_creation() { - let clustering_info = ClusteringInfo { - clustering_columns: vec!["col1".to_string(), "col2".to_string()], - cluster_count: 5, - avg_files_per_cluster: 20.0, - avg_cluster_size_bytes: 2000.0, - }; - - assert_eq!(clustering_info.clustering_columns, vec!["col1", "col2"]); - assert_eq!(clustering_info.cluster_count, 5); - assert_eq!(clustering_info.avg_files_per_cluster, 20.0); - assert_eq!(clustering_info.avg_cluster_size_bytes, 2000.0); - } - - #[test] - fn test_file_size_distribution_creation() { - let distribution = FileSizeDistribution { - small_files: 10, - medium_files: 20, - large_files: 5, - very_large_files: 1, - }; - - assert_eq!(distribution.small_files, 10); - assert_eq!(distribution.medium_files, 20); - assert_eq!(distribution.large_files, 5); - assert_eq!(distribution.very_large_files, 1); - } - - #[test] - fn test_deletion_vector_metrics_creation() { - let dv_metrics = DeletionVectorMetrics { - deletion_vector_count: 5, - total_deletion_vector_size_bytes: 1024 * 1024, - avg_deletion_vector_size_bytes: 204.8 * 1024.0, - deletion_vector_age_days: 10.0, - deleted_rows_count: 1000, - deletion_vector_impact_score: 0.5, - }; - - assert_eq!(dv_metrics.deletion_vector_count, 5); - assert_eq!(dv_metrics.total_deletion_vector_size_bytes, 1024 * 1024); - assert_eq!(dv_metrics.avg_deletion_vector_size_bytes, 204.8 * 1024.0); - assert_eq!(dv_metrics.deletion_vector_age_days, 10.0); - assert_eq!(dv_metrics.deleted_rows_count, 1000); - assert_eq!(dv_metrics.deletion_vector_impact_score, 0.5); - } - - #[test] - fn test_schema_evolution_metrics_creation() { - let schema_metrics = SchemaEvolutionMetrics { - total_schema_changes: 10, - breaking_changes: 2, - non_breaking_changes: 8, - schema_stability_score: 0.8, - days_since_last_change: 5.0, - schema_change_frequency: 0.1, - current_schema_version: 10, - }; - - assert_eq!(schema_metrics.total_schema_changes, 10); - assert_eq!(schema_metrics.breaking_changes, 2); - assert_eq!(schema_metrics.non_breaking_changes, 8); - assert_eq!(schema_metrics.schema_stability_score, 0.8); - assert_eq!(schema_metrics.days_since_last_change, 5.0); - assert_eq!(schema_metrics.schema_change_frequency, 0.1); - assert_eq!(schema_metrics.current_schema_version, 10); - } - - #[test] - fn test_time_travel_metrics_creation() { - let tt_metrics = TimeTravelMetrics { - total_snapshots: 50, - oldest_snapshot_age_days: 30.0, - newest_snapshot_age_days: 0.0, - total_historical_size_bytes: 5 * 1024 * 1024 * 1024, - avg_snapshot_size_bytes: 100.0 * 1024.0 * 1024.0, - storage_cost_impact_score: 0.3, - retention_efficiency_score: 0.7, - recommended_retention_days: 14, - }; - - assert_eq!(tt_metrics.total_snapshots, 50); - assert_eq!(tt_metrics.oldest_snapshot_age_days, 30.0); - assert_eq!(tt_metrics.newest_snapshot_age_days, 0.0); - assert_eq!(tt_metrics.total_historical_size_bytes, 5 * 1024 * 1024 * 1024); - assert_eq!(tt_metrics.avg_snapshot_size_bytes, 100.0 * 1024.0 * 1024.0); - assert_eq!(tt_metrics.storage_cost_impact_score, 0.3); - assert_eq!(tt_metrics.retention_efficiency_score, 0.7); - assert_eq!(tt_metrics.recommended_retention_days, 14); - } - - #[test] - fn test_table_constraints_metrics_creation() { - let constraint_metrics = TableConstraintsMetrics { - total_constraints: 8, - check_constraints: 3, - not_null_constraints: 4, - unique_constraints: 1, - foreign_key_constraints: 0, - constraint_violation_risk: 0.2, - data_quality_score: 0.9, - constraint_coverage_score: 0.8, - }; - - assert_eq!(constraint_metrics.total_constraints, 8); - assert_eq!(constraint_metrics.check_constraints, 3); - assert_eq!(constraint_metrics.not_null_constraints, 4); - assert_eq!(constraint_metrics.unique_constraints, 1); - assert_eq!(constraint_metrics.foreign_key_constraints, 0); - assert_eq!(constraint_metrics.constraint_violation_risk, 0.2); - assert_eq!(constraint_metrics.data_quality_score, 0.9); - assert_eq!(constraint_metrics.constraint_coverage_score, 0.8); - } - - #[test] - fn test_file_compaction_metrics_creation() { - let compaction_metrics = FileCompactionMetrics { - compaction_opportunity_score: 0.7, - small_files_count: 25, - small_files_size_bytes: 50 * 1024 * 1024, - potential_compaction_files: 25, - estimated_compaction_savings_bytes: 10 * 1024 * 1024, - recommended_target_file_size_bytes: 128 * 1024 * 1024, - compaction_priority: "medium".to_string(), - z_order_opportunity: true, - z_order_columns: vec!["col1".to_string(), "col2".to_string()], - }; - - assert_eq!(compaction_metrics.compaction_opportunity_score, 0.7); - assert_eq!(compaction_metrics.small_files_count, 25); - assert_eq!(compaction_metrics.small_files_size_bytes, 50 * 1024 * 1024); - assert_eq!(compaction_metrics.potential_compaction_files, 25); - assert_eq!(compaction_metrics.estimated_compaction_savings_bytes, 10 * 1024 * 1024); - assert_eq!(compaction_metrics.recommended_target_file_size_bytes, 128 * 1024 * 1024); - assert_eq!(compaction_metrics.compaction_priority, "medium"); - assert!(compaction_metrics.z_order_opportunity); - assert_eq!(compaction_metrics.z_order_columns, vec!["col1", "col2"]); - } -} diff --git a/tests/test_drainage.py b/tests/test_drainage.py index dcc5481..8fbb57a 100644 --- a/tests/test_drainage.py +++ b/tests/test_drainage.py @@ -10,6 +10,7 @@ import sys import os from unittest.mock import patch, MagicMock +import pytest # Add the parent directory to the path so we can import drainage sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -21,6 +22,7 @@ drainage = None +@pytest.mark.unit class TestDrainageModule(unittest.TestCase): """Test cases for the drainage module.""" @@ -415,6 +417,7 @@ def test_snapshot_health_structure(self): ) +@pytest.mark.integration class TestDrainageIntegration(unittest.TestCase): """Integration tests for the drainage module."""