From fbad4fbfad5384f6dd2df6e325ba2ef4c1426d55 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 00:03:27 -0700 Subject: [PATCH 01/31] feat(rust): crate skeleton with shared types and traits for the Rust port --- .gitignore | 1 + rust/Cargo.lock | 1164 +++++++++++++++++++++++++++++++++ rust/Cargo.toml | 43 ++ rust/src/cancel.rs | 98 +++ rust/src/clock.rs | 80 +++ rust/src/codec.rs | 98 +++ rust/src/error.rs | 115 ++++ rust/src/identity.rs | 133 ++++ rust/src/lib.rs | 38 ++ rust/src/limits.rs | 36 + rust/src/local.rs | 74 +++ rust/src/observe.rs | 266 ++++++++ rust/src/policy.rs | 222 +++++++ rust/src/protocol/envelope.rs | 71 ++ rust/src/protocol/frame.rs | 67 ++ rust/src/protocol/mod.rs | 16 + rust/src/protocol/text.rs | 8 + rust/src/remote.rs | 126 ++++ rust/src/spawn.rs | 29 + rust/tests/conformance.rs | 3 + 20 files changed, 2688 insertions(+) create mode 100644 rust/Cargo.lock create mode 100644 rust/Cargo.toml create mode 100644 rust/src/cancel.rs create mode 100644 rust/src/clock.rs create mode 100644 rust/src/codec.rs create mode 100644 rust/src/error.rs create mode 100644 rust/src/identity.rs create mode 100644 rust/src/lib.rs create mode 100644 rust/src/limits.rs create mode 100644 rust/src/local.rs create mode 100644 rust/src/observe.rs create mode 100644 rust/src/policy.rs create mode 100644 rust/src/protocol/envelope.rs create mode 100644 rust/src/protocol/frame.rs create mode 100644 rust/src/protocol/mod.rs create mode 100644 rust/src/protocol/text.rs create mode 100644 rust/src/remote.rs create mode 100644 rust/src/spawn.rs create mode 100644 rust/tests/conformance.rs diff --git a/.gitignore b/.gitignore index 76a65fcb..5877d6cb 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ docs/.vitepress/cache/ .idea/ .vscode/ *.log +rust/target/ diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 00000000..08e728c3 --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,1164 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "arcstr" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03918c3dbd7701a85c6b9887732e2921175f26c350b4563841d0958c21d57e6d" + +[[package]] +name = "async-lock" +version = "3.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f7f2596bd5b78a9fec8088ccd89180d7f9f55b94b0576823bbbdc72ee8311" +dependencies = [ + "event-listener", + "event-listener-strategy", + "pin-project-lite", +] + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54413ede23c2daf518f35156dfde027feb2374004d63bd497f983c8db9c0e313" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "chacha20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "futures-core", + "memchr", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + +[[package]] +name = "crc16" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff" + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "dialcache" +version = "0.1.0" +dependencies = [ + "futures", + "hex", + "log", + "lru", + "parking_lot", + "prometheus", + "redis", + "serde", + "serde_json", + "sha1", + "thiserror 2.0.20", + "tokio", + "zstd", + "zstd-safe 7.3.0", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef25905e51abafe4dcea6c15fec58c57b601cdbd0ee53d22ea1d3016c587d39b" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a31d2a3fbaaeb2af2368bbdd904aa8e812d3c04a1ee10d3171f52d556e5d0a3" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f9e3d69d39e4862ffed03ed071a76f9a13ba1d9109d355b0f0aa6b15e393c4" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d699e522242e69e3003b94ecc1f960f3a5e015aa7c5d7486e65ad01dd94f5e" + +[[package]] +name = "futures-executor" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "031b47cf1a3c6cc8bc2fc76cd437f521619387907d469316e7c0bc278f1f5432" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" + +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "futures-sink" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" + +[[package]] +name = "futures-task" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd" + +[[package]] +name = "futures-util" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d50a92467f8ba5dd6e3ee5d4bd04d73ab2e4e1c44474a0674821dfce14b79bc" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + +[[package]] +name = "icu_collections" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" + +[[package]] +name = "icu_properties" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" + +[[package]] +name = "icu_provider" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "litemap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d9d19d1d6efa0109d2f65ff4c85cddd50bd572e5a00127ab10987290bcefae" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "lru" +version = "0.18.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff9840bcc50b71349309900da0ce7279aa336ae71d73250b07998932c7d97c25" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "mio" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "potential_utf" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d83eb9bc6d8e5cf568e7a1101d60ee05e81ed50ea106026f3d18deeb046d7661" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "prometheus" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ca5326d8d0b950a9acd87e6a3f94745394f62e4dae1b1ee22b2bc0c394af43a" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror 2.0.20", +] + +[[package]] +name = "protobuf" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf-support" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" +dependencies = [ + "thiserror 1.0.69", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redis" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed" +dependencies = [ + "arcstr", + "async-lock", + "bytes", + "cfg-if", + "combine", + "crc16", + "futures-channel", + "futures-util", + "itoa", + "log", + "num-bigint", + "percent-encoding", + "pin-project-lite", + "rand", + "ryu", + "sha1_smol", + "socket2", + "tokio", + "tokio-util", + "url", + "xxhash-rust", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl 1.0.69", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl 2.0.20", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tinystr" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "libc", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "writeable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" + +[[package]] +name = "xxhash-rust" +version = "0.8.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aee1b19627c7c60102ab80d3a9cbe18de90bfe03bfa6c3715447681f0e8c8af6" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33811428bee40dbceb6d545e95754741d17a6aef9a4849f0fd62e2ba4f412a78" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", + "synstructure", +] + +[[package]] +name = "zerotrie" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ea269c3bd32f0a32c321907a2ae912ba6f4649bb0fc764a15627e99a7095a3f" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zstd" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf06bd8162af0734b344780deb55b42a2429ae430870d13fcc12f238e880fe6e" +dependencies = [ + "zstd-safe 8.0.0", +] + +[[package]] +name = "zstd-safe" +version = "7.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-safe" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae42c0555055784c70058d19ba8e275528e8a99a706684868ace5da4e716a4ab" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.1.0+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 00000000..2f68cc6d --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "dialcache" +version = "0.1.0" +edition = "2021" +rust-version = "1.85" +license = "MIT" +description = "Explicitly enabled, layered caching with runtime rollout, request coalescing, tracked invalidation, stale recovery and shadow validation. Rust port of DialCache." +repository = "https://github.com/lan17/DialCache" +keywords = ["cache", "redis", "async"] +categories = ["caching"] +publish = false + +[features] +default = ["tokio"] +tokio = ["dep:tokio"] +redis = ["dep:redis", "dep:sha1", "dep:hex"] +prometheus = ["dep:prometheus"] + +[dependencies] +futures = "0.3" +parking_lot = "0.12" +lru = "0.18" +serde = { version = "1", features = ["derive"] } +serde_json = { version = "1", features = ["float_roundtrip"] } +zstd = "0.14" +zstd-safe = "7" +thiserror = "2" +log = "0.4" +tokio = { version = "1", features = ["rt", "time"], optional = true } +redis = { version = "1", features = ["tokio-comp", "cluster-async", "script"], optional = true } +sha1 = { version = "0.11", optional = true } +hex = { version = "0.4", optional = true } +prometheus = { version = "0.14", optional = true } + +[dev-dependencies] +tokio = { version = "1", features = ["full"] } +hex = "0.4" +serde_json = "1" + +[[test]] +name = "conformance" +path = "tests/conformance.rs" +harness = false diff --git a/rust/src/cancel.rs b/rust/src/cancel.rs new file mode 100644 index 00000000..aec4ac60 --- /dev/null +++ b/rust/src/cancel.rs @@ -0,0 +1,98 @@ +//! Cooperative cancellation for remote reads. + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll, Waker}; + +use parking_lot::Mutex; + +#[derive(Default)] +struct Inner { + cancelled: bool, + wakers: Vec, + callbacks: Vec>, +} + +/// A cancellation request handed to remote adapters with each read. +/// +/// The cache cancels the token once the read deadline passes. Cancellation is +/// a request; it does not prove that a dispatched command stopped. +#[derive(Clone, Default)] +pub struct CancelToken { + inner: Arc>, +} + +impl std::fmt::Debug for CancelToken { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CancelToken").field("cancelled", &self.is_cancelled()).finish() + } +} + +impl CancelToken { + pub fn new() -> Self { + Self::default() + } + + /// Whether cancellation was requested. + pub fn is_cancelled(&self) -> bool { + self.inner.lock().cancelled + } + + /// Request cancellation once. Later calls do nothing. + pub fn cancel(&self) { + let (wakers, callbacks) = { + let mut inner = self.inner.lock(); + if inner.cancelled { + return; + } + inner.cancelled = true; + (std::mem::take(&mut inner.wakers), std::mem::take(&mut inner.callbacks)) + }; + for waker in wakers { + waker.wake(); + } + for callback in callbacks { + callback(); + } + } + + /// Run `callback` when cancellation is requested, or immediately if it already was. + pub fn on_cancel(&self, callback: impl FnOnce() + Send + 'static) { + let mut callback = Some(Box::new(callback) as Box); + { + let mut inner = self.inner.lock(); + if !inner.cancelled { + inner.callbacks.push(callback.take().expect("callback present")); + } + } + if let Some(callback) = callback { + callback(); + } + } + + /// Completes once cancellation is requested. + pub fn cancelled(&self) -> Cancelled { + Cancelled { token: self.clone() } + } +} + +/// Future returned by [`CancelToken::cancelled`]. +pub struct Cancelled { + token: CancelToken, +} + +impl Future for Cancelled { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + let mut inner = self.token.inner.lock(); + if inner.cancelled { + return Poll::Ready(()); + } + if !inner.wakers.iter().any(|w| w.will_wake(cx.waker())) { + inner.wakers.push(cx.waker().clone()); + } + Poll::Pending + } +} diff --git a/rust/src/clock.rs b/rust/src/clock.rs new file mode 100644 index 00000000..c089fc50 --- /dev/null +++ b/rust/src/clock.rs @@ -0,0 +1,80 @@ +//! Wall and elapsed clocks. + +use std::time::Duration; + +use futures::future::BoxFuture; + +/// Separates wall timestamps from monotonic elapsed time and timers. +/// +/// Wall time stamps frames and invalidation markers. Elapsed time governs +/// deadlines and process-local expiry. Timers deliver deadline callbacks; a +/// controlled clock may deliver them on its own schedule. +pub trait Clock: Send + Sync + 'static { + /// Epoch milliseconds from the application wall clock. + fn wall_ms(&self) -> i64; + /// Monotonic elapsed time since this clock's origin, with native precision. + fn elapsed(&self) -> Duration; + /// Whole-millisecond elapsed reading used by process-local storage. + /// + /// Insertion and lookup share this integer grid, so an entry inserted at + /// 0.7 ms with a 1,000 ms TTL expires when this reading reaches 1,000. + fn elapsed_ms(&self) -> i64 { + self.elapsed().as_millis().min(i64::MAX as u128) as i64 + } + /// A timer that completes once `duration` of this clock's time has passed. + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()>; +} + +#[cfg(feature = "tokio")] +mod system { + use super::Clock; + use futures::future::BoxFuture; + use std::sync::OnceLock; + use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + + static PROCESS_ORIGIN: OnceLock = OnceLock::new(); + + /// The default clock: system wall time, a monotonic origin aligned to the + /// process-wide millisecond grid, and tokio timers. + #[derive(Debug, Clone)] + pub struct SystemClock { + origin: Instant, + } + + impl SystemClock { + /// A clock whose whole-millisecond grid is shared by every default instance. + pub fn new() -> Self { + let process = *PROCESS_ORIGIN.get_or_init(Instant::now); + let now = Instant::now(); + let phase = now.saturating_duration_since(process).as_nanos() % 1_000_000; + let origin = now.checked_sub(Duration::from_nanos(phase as u64)).unwrap_or(now); + SystemClock { origin } + } + } + + impl Default for SystemClock { + fn default() -> Self { + Self::new() + } + } + + impl Clock for SystemClock { + fn wall_ms(&self) -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis().min(i64::MAX as u128) as i64) + .unwrap_or(0) + } + + fn elapsed(&self) -> Duration { + self.origin.elapsed() + } + + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { + Box::pin(tokio::time::sleep(duration)) + } + } +} + +#[cfg(feature = "tokio")] +pub use system::SystemClock; diff --git a/rust/src/codec.rs b/rust/src/codec.rs new file mode 100644 index 00000000..dcc77b38 --- /dev/null +++ b/rust/src/codec.rs @@ -0,0 +1,98 @@ +//! Serialized payloads and value codecs. + +use std::borrow::Cow; + +use futures::future::BoxFuture; +use serde::de::DeserializeOwned; +use serde::Serialize; + +use crate::error::BoxError; + +/// Serializer output: UTF-8 text or opaque binary bytes. +/// +/// Text payloads are stored with the frame's UTF-8 tag; binary payloads keep +/// their exact bytes. Compression envelopes wrap either form transparently. +#[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] +pub struct Payload { + pub bytes: Vec, + /// `true` for binary output, `false` for UTF-8 text. + pub binary: bool, +} + +impl Payload { + pub fn text(text: impl Into) -> Self { + Payload { bytes: text.into().into_bytes(), binary: false } + } + + pub fn binary(bytes: impl Into>) -> Self { + Payload { bytes: bytes.into(), binary: true } + } + + pub fn len(&self) -> usize { + self.bytes.len() + } + + pub fn is_empty(&self) -> bool { + self.bytes.is_empty() + } + + /// The bytes as text, replacing ill-formed UTF-8 in binary payloads. + pub fn as_text(&self) -> Cow<'_, str> { + String::from_utf8_lossy(&self.bytes) + } +} + +/// Encodes cached values to payloads and decodes them back. +/// +/// Implement [`SyncCodec`] for ordinary serializers; it is adapted to this +/// asynchronous form automatically. Implement this trait directly only when +/// encoding or decoding must await. Every decode must return an independent +/// value: the cache may retain one payload and decode it more than once. +pub trait Codec: Send + Sync + 'static { + fn encode<'a>(&'a self, value: &'a T) -> BoxFuture<'a, Result>; + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result>; +} + +/// A synchronous codec. Blanket-adapted to [`Codec`]. +pub trait SyncCodec: Send + Sync + 'static { + fn encode(&self, value: &T) -> Result; + fn decode(&self, payload: Payload) -> Result; +} + +impl> Codec for C { + fn encode<'a>(&'a self, value: &'a T) -> BoxFuture<'a, Result> { + let result = SyncCodec::encode(self, value); + Box::pin(std::future::ready(result)) + } + + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result> { + let result = SyncCodec::decode(self, payload); + Box::pin(std::future::ready(result)) + } +} + +/// Payload text that TypeScript writes for an `undefined` value. +pub const JSON_UNDEFINED_SENTINEL: &str = "__dialcache_json_undefined_v1__"; + +/// The default codec: `serde_json` text. +/// +/// Values written by TypeScript's default serializer decode as long as they +/// are valid JSON for the destination type. The TypeScript `undefined` +/// sentinel decodes as JSON `null`, so an `Option` destination reads it as +/// `None` and any other destination fails open to the source. +#[derive(Debug, Clone, Copy, Default)] +pub struct JsonCodec; + +impl SyncCodec for JsonCodec { + fn encode(&self, value: &T) -> Result { + Ok(Payload::text(serde_json::to_string(value)?)) + } + + fn decode(&self, payload: Payload) -> Result { + let text = payload.as_text(); + if text == JSON_UNDEFINED_SENTINEL { + return Ok(T::deserialize(serde_json::Value::Null)?); + } + Ok(serde_json::from_str(&text)?) + } +} diff --git a/rust/src/error.rs b/rust/src/error.rs new file mode 100644 index 00000000..e4762018 --- /dev/null +++ b/rust/src/error.rs @@ -0,0 +1,115 @@ +//! Public error types. + +use std::fmt; +use std::sync::Arc; + +/// Boxed error returned by sources, codecs, providers and adapters. +pub type BoxError = Box; +/// Shared error handed to every caller that joined one execution. +pub type SharedError = Arc; + +/// Every failure a cached call or maintenance operation can report. +/// +/// Cache plumbing fails open, so a call fails only when its source fails, +/// its source deadline elapses, a callback panics, or static configuration +/// is rejected before execution. Maintenance operations surface their own +/// remote failures. +#[derive(Debug, Clone)] +pub enum Error { + /// The source returned an error. Every coalesced caller receives the same + /// shared error, so identity comparisons through [`Arc::ptr_eq`] work. + Source(SharedError), + /// The source did not settle before its deadline. Followers of one + /// execution share the leader's error instance. + FallbackTimeout(Arc), + /// A source, codec, provider or comparator panicked. + Panic(Arc), + /// Static operation or instance configuration was rejected before execution. + Config(ConfigError), + /// Invalidation was requested without a configured remote adapter. + MissingRemote, + /// An explicit maintenance mutation failed at the remote adapter. + Remote(SharedError), +} + +impl Error { + /// The source error, when the source itself failed. + pub fn source_error(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> { + match self { + Error::Source(error) => Some(error.as_ref()), + _ => None, + } + } + + /// Whether this is the library's own source deadline error. + pub fn is_fallback_timeout(&self) -> bool { + matches!(self, Error::FallbackTimeout(_)) + } + + pub(crate) fn from_box(error: BoxError) -> Error { + Error::Source(Arc::from(error)) + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Error::Source(error) => write!(f, "DialCache source failed: {error}"), + Error::FallbackTimeout(timeout) => timeout.fmt(f), + Error::Panic(message) => write!(f, "DialCache callback panicked: {message}"), + Error::Config(error) => error.fmt(f), + Error::MissingRemote => f.write_str("DialCache invalidation requires a configured remote adapter"), + Error::Remote(error) => write!(f, "DialCache remote maintenance failed: {error}"), + } + } +} + +impl std::error::Error for Error { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + Error::Source(error) | Error::Remote(error) => Some(error.as_ref()), + Error::FallbackTimeout(timeout) => Some(timeout.as_ref()), + Error::Config(error) => Some(error), + _ => None, + } + } +} + +impl From for Error { + fn from(error: ConfigError) -> Self { + Error::Config(error) + } +} + +/// The source deadline elapsed before the source settled. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("DialCache fallback for use case {use_case:?} timed out after {timeout_ms} ms")] +pub struct FallbackTimeout { + pub use_case: String, + pub timeout_ms: u64, +} + +/// The remote read deadline elapsed before the adapter answered. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("DialCache remote read for use case {use_case:?} timed out after {timeout_ms} ms")] +pub struct RemoteReadTimeout { + pub use_case: String, + pub timeout_ms: u64, +} + +/// Static configuration rejected at construction or registration. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ConfigError { + #[error("{0}")] + Invalid(String), + #[error("DialCache use case name is reserved: {0}")] + ReservedUseCase(String), + #[error("DialCache use case already registered: {0}")] + UseCaseAlreadyRegistered(String), +} + +impl ConfigError { + pub(crate) fn invalid(message: impl Into) -> Self { + ConfigError::Invalid(message.into()) + } +} diff --git a/rust/src/identity.rs b/rust/src/identity.rs new file mode 100644 index 00000000..f232b9fb --- /dev/null +++ b/rust/src/identity.rs @@ -0,0 +1,133 @@ +//! Logical identities, shared cache keys and rollout cohorts. +//! +//! Key construction follows the portable protocol (W01–W03): URI component +//! escaping, tracked entity hash tags, ordered argument pairs, the frame key +//! suffix, and FNV-1a cohorts over UTF-16 code units. + +/// A normalized logical identity. Ordered arguments retain caller order; +/// use [`normalize_args`] to build them from a host-language record. +#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +pub struct Identity { + #[serde(default)] + pub namespace: String, + #[serde(rename = "keyType")] + pub key_type: String, + pub id: String, + #[serde(rename = "useCase")] + pub use_case: String, + #[serde(rename = "trackForInvalidation", default)] + pub tracked: bool, + #[serde(default)] + pub args: Vec<(String, String)>, +} + +impl Identity { + pub fn new(key_type: impl Into, id: impl Into, use_case: impl Into) -> Self { + Identity { + namespace: String::new(), + key_type: key_type.into(), + id: id.into(), + use_case: use_case.into(), + tracked: false, + args: Vec::new(), + } + } + + pub fn tracked(mut self, tracked: bool) -> Self { + self.tracked = tracked; + self + } + + pub fn namespace(mut self, namespace: impl Into) -> Self { + self.namespace = namespace.into(); + self + } + + pub fn args(mut self, args: Vec<(String, String)>) -> Self { + self.args = args; + self + } + + /// The logical key, the stored value key and, for tracked identities, the watermark key. + pub fn keys(&self) -> Result { + keys(self) + } +} + +/// Derived key strings of one identity. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Keys { + /// Logical key: `namespace:keyType:id[?args]#useCase`, hash-tagged when tracked. + pub logical: String, + /// Stored value key: the logical key plus the frame suffix. + pub value: String, + /// Entity watermark key for tracked identities. + pub watermark: Option, +} + +/// Why an identity cannot form a key. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum IdentityError { + #[error("DialCache identity component contains a reserved hash-tag delimiter")] + ReservedDelimiter, + #[error("DialCache identity contains an unsupported argument value: {0}")] + UnsupportedArgument(String), +} + +/// A host scalar accepted as a key argument. +#[derive(Debug, Clone, PartialEq)] +pub enum ArgValue { + /// Omitted from the key (JavaScript `undefined`). + Absent, + /// JSON null, spelled `null`. + Null, + Bool(bool), + /// Spelled with JavaScript `Number` formatting. + Number(f64), + Int(i64), + /// An arbitrary-precision integer as decimal text. + BigInt(String), + Str(String), +} + +/// Normalize a record of key arguments: omit absent values, spell scalars +/// the JavaScript way, and sort names by UTF-16 code units. +pub fn normalize_args(args: I) -> Result, IdentityError> +where + I: IntoIterator, + K: Into, +{ + let _ = args; + todo!("normalize_args") +} + +/// Build every key of an identity. +pub fn keys(identity: &Identity) -> Result { + let _ = identity; + todo!("keys") +} + +/// `encodeURIComponent` escaping of one component. +pub fn escape_component(component: &str) -> String { + let _ = component; + todo!("escape_component") +} + +/// Deterministic rollout sample in `[0, 100)` for a logical key and layer or +/// shadow discriminator: FNV-1a over the UTF-16 units of `key:discriminator`. +pub fn cohort(logical_key: &str, discriminator: &str) -> f64 { + let _ = (logical_key, discriminator); + todo!("cohort") +} + +/// Spell a number the way JavaScript's `String(number)` does. +pub fn js_number_to_string(value: f64) -> String { + let _ = value; + todo!("js_number_to_string") +} + +/// Compare two strings by UTF-16 code units, as JavaScript's `<` does. +pub fn compare_utf16(left: &str, right: &str) -> std::cmp::Ordering { + let _ = (left, right); + todo!("compare_utf16") +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 00000000..44819196 --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,38 @@ +//! DialCache: explicitly enabled, layered caching with runtime rollout, +//! request coalescing, tracked invalidation, stale recovery and shadow +//! validation. +//! +//! This crate is a port of the TypeScript library. Its portable behavior is +//! defined by the Quint models under `formal/` and checked by replaying the +//! same generated histories through the real API. + +#![forbid(unsafe_code)] +#![warn(missing_debug_implementations)] + +pub mod cancel; +pub mod clock; +pub mod codec; +pub mod error; +pub mod identity; +pub mod limits; +pub mod local; +pub mod observe; +pub mod policy; +pub mod protocol; +pub mod remote; +pub mod spawn; + +pub use cancel::CancelToken; +pub use clock::Clock; +#[cfg(feature = "tokio")] +pub use clock::SystemClock; +pub use codec::{Codec, JsonCodec, Payload, SyncCodec}; +pub use error::{BoxError, ConfigError, Error, FallbackTimeout, RemoteReadTimeout, SharedError}; +pub use identity::{normalize_args, ArgValue, Identity, IdentityError, Keys}; +pub use local::{LocalEntry, LocalStore, LruLocalStore, StoredValue}; +pub use observe::{Event, Labels, LogEvent, LogLevel, Logger, Observer, ShadowMismatchDetails}; +pub use policy::{Policy, PolicyDefaults, PolicyError, ResolvedPolicy, RuntimePolicy, ShadowPolicy}; +pub use remote::{Frame, InvalidateRequest, MissReason, ReadContext, ReadRequest, ReadResult, Remote, WriteRequest}; +pub use spawn::Spawner; +#[cfg(feature = "tokio")] +pub use spawn::TokioSpawner; diff --git a/rust/src/limits.rs b/rust/src/limits.rs new file mode 100644 index 00000000..50214a76 --- /dev/null +++ b/rust/src/limits.rs @@ -0,0 +1,36 @@ +//! Numeric domains shared by every DialCache implementation. + +/// Largest integer JavaScript represents exactly; timestamps and counters +/// beyond it are rejected rather than rounded. +pub const MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991; +/// Fixed 365-day ceiling shared by cache TTLs and invalidation buffers, in milliseconds. +pub const MAX_SUPPORTED_DURATION_MS: u64 = 365 * 24 * 60 * 60 * 1_000; +/// Cache TTL ceiling in whole seconds. +pub const MAX_CACHE_TTL_SEC: u64 = MAX_SUPPORTED_DURATION_MS / 1_000; +/// Tracked Redis values are physically retained for at most one hour so +/// invalidation markers can safely age out. +pub const MAX_TRACKED_VALUE_TTL_MS: u64 = 60 * 60 * 1_000; +/// Largest source or read deadline, in milliseconds. +pub const MAX_DEADLINE_MS: u64 = 2_147_483_647; +/// Library default remote read budget. +pub const DEFAULT_REMOTE_READ_TIMEOUT_MS: u64 = 50; +/// Library default source budget. +pub const DEFAULT_FALLBACK_TIMEOUT_MS: u64 = 60_000; +/// Default process-local capacity per instance. +pub const DEFAULT_LOCAL_CAPACITY: usize = 10_000; +/// Default concurrent shadow jobs per instance. +pub const DEFAULT_SHADOW_MAX_IN_FLIGHT: usize = 1; +/// Ceiling on one decompressed payload (Redis's own value limit). +pub const MAX_DECOMPRESSED_BYTES: usize = 512 * 1024 * 1024; +/// Default compression threshold in serialized bytes. +pub const DEFAULT_COMPRESSION_THRESHOLD_BYTES: usize = 4096; +/// Default zstd level. +pub const DEFAULT_ZSTD_LEVEL: i32 = 3; +/// Reserved use case name owned by invalidation watermarks. +pub const WATERMARK_USE_CASE: &str = "watermark"; +/// Key suffix of every stored value frame. +pub const FRAME_KEY_SUFFIX: &str = ":dialcache-frame-v1"; +/// Minimum watermark retention: twice the tracked value cap. +pub const MIN_WATERMARK_TTL_MS: u64 = 2 * MAX_TRACKED_VALUE_TTL_MS; +/// Slack added to derived watermark retention. +pub const WATERMARK_TTL_MARGIN_MS: u64 = 60_000; diff --git a/rust/src/local.rs b/rust/src/local.rs new file mode 100644 index 00000000..5770c1e7 --- /dev/null +++ b/rust/src/local.rs @@ -0,0 +1,74 @@ +//! Process-local storage. + +use std::any::Any; +use std::num::NonZeroUsize; +use std::sync::Arc; + +use crate::error::BoxError; + +/// A type-erased cached value shared by reference. +pub type StoredValue = Arc; + +/// One process-local entry with its insertion-time TTL. +#[derive(Clone)] +pub struct LocalEntry { + pub value: StoredValue, + /// Whole-millisecond elapsed reading at insertion. + pub inserted_ms: i64, + pub ttl_ms: i64, +} + +/// Process-local storage for one cache instance. +/// +/// The default [`LruLocalStore`] evicts the least recently used entry at +/// capacity regardless of its remaining TTL, checks expiry lazily on read +/// against whole elapsed milliseconds, promotes on read and write, and never +/// renews a TTL on read. Implementations must preserve those rules. +pub trait LocalStore: Send + 'static { + /// Return a live entry's value, promoting it. Expired entries are removed + /// and reported as absent without promotion. + fn get(&mut self, key: &str, now_ms: i64) -> Result, BoxError>; + /// Insert or replace an entry, promoting it and evicting at capacity. + fn put(&mut self, key: String, entry: LocalEntry) -> Result<(), BoxError>; +} + +/// The default LRU store. +pub struct LruLocalStore { + entries: lru::LruCache, +} + +impl LruLocalStore { + /// A store holding at most `capacity` entries. Capacity must be positive. + pub fn new(capacity: NonZeroUsize) -> Self { + LruLocalStore { entries: lru::LruCache::new(capacity) } + } + + pub fn len(&self) -> usize { + self.entries.len() + } + + pub fn is_empty(&self) -> bool { + self.entries.is_empty() + } +} + +impl LocalStore for LruLocalStore { + fn get(&mut self, key: &str, now_ms: i64) -> Result, BoxError> { + // Peek, check freshness, then promote: an expired entry leaves the LRU + // order untouched apart from its own removal. + let expired = match self.entries.peek(key) { + None => return Ok(None), + Some(entry) => now_ms.saturating_sub(entry.inserted_ms) >= entry.ttl_ms, + }; + if expired { + self.entries.pop(key); + return Ok(None); + } + Ok(self.entries.get(key).map(|entry| entry.value.clone())) + } + + fn put(&mut self, key: String, entry: LocalEntry) -> Result<(), BoxError> { + self.entries.put(key, entry); + Ok(()) + } +} diff --git a/rust/src/observe.rs b/rust/src/observe.rs new file mode 100644 index 00000000..75255d1a --- /dev/null +++ b/rust/src/observe.rs @@ -0,0 +1,266 @@ +//! Backend-neutral diagnostics: metric events and structured log events. + +use std::fmt; +use std::sync::Arc; + +use crate::error::BoxError; +pub use crate::remote::MissReason; + +macro_rules! str_enum { + ($(#[$meta:meta])* $name:ident { $($variant:ident => $text:literal),+ $(,)? }) => { + $(#[$meta])* + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub enum $name { $($variant),+ } + impl $name { + pub fn as_str(self) -> &'static str { + match self { $($name::$variant => $text),+ } + } + } + impl fmt::Display for $name { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.write_str(self.as_str()) } + } + }; +} + +str_enum! { + /// The cache layer a diagnostic refers to. + Layer { + Noop => "noop", + RequestLocal => "request_local", + Local => "local", + Remote => "remote", + RemoteShadow => "remote_shadow", + } +} + +str_enum! { + /// Where an in-flight execution was shared. + CoalescingScope { RequestLocal => "request_local", Process => "process" } +} + +str_enum! { + /// Why a cache layer was skipped. + DisabledReason { + Context => "context", + PolicyDisabled => "policy_disabled", + InvalidTtl => "invalid_ttl", + InvalidRamp => "invalid_ramp", + RampedDown => "ramped_down", + ConfigError => "config_error", + } +} + +str_enum! { + /// Stable failure sites. + ErrorKind { + KeyConstruction => "key_construction", + ConfigResolution => "config_resolution", + CacheRead => "cache_read", + CacheReadTimeout => "cache_read_timeout", + CacheWrite => "cache_write", + TrackedTtlClamped => "tracked_ttl_clamped", + SerializationLoad => "serialization_load", + SerializationDump => "serialization_dump", + Compression => "compression", + Invalidation => "invalidation", + Fallback => "fallback", + } +} + +str_enum! { + /// Terminal outcomes of sampled shadow validation. + ShadowOutcome { + Match => "match", + Mismatch => "mismatch", + Superseded => "superseded", + Filled => "filled", + FillFenced => "fill_fenced", + FillError => "fill_error", + RedisError => "redis_error", + SourceError => "source_error", + DeserializationError => "deserialization_error", + ComparisonError => "comparison_error", + ConfirmationError => "confirmation_error", + Timeout => "timeout", + Dropped => "dropped", + } +} + +str_enum! { + /// Terminal outcomes of stale-on-error recovery. + RecoveryOutcome { + Served => "served", + Miss => "miss", + DeserializationError => "deserialization_error", + } +} + +str_enum! { + /// Compression outcomes for writes and reads. + CompressionOutcome { + Compressed => "compressed", + BelowThreshold => "below_threshold", + NotSmaller => "not_smaller", + WriteOverLimit => "write_over_limit", + Decompressed => "decompressed", + FallbackRaw => "fallback_raw", + ReadOverLimit => "read_over_limit", + } +} + +str_enum! { + SerializationOperation { Dump => "dump", Load => "load" } +} + +str_enum! { + CompressionOperation { Compress => "compress", Decompress => "decompress" } +} + +/// Labels shared by layer-scoped events. Logical keys are never labels. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Labels { + pub namespace: Arc, + pub use_case: Arc, + pub key_type: Arc, + pub layer: Layer, +} + +/// Labels of outcome-scoped events (shadow validation and recovery). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct OutcomeLabels { + pub namespace: Arc, + pub use_case: Arc, + pub key_type: Arc, +} + +/// One public diagnostic. Counters carry no value; timers report seconds; +/// sizes report bytes. +#[derive(Debug, Clone, PartialEq)] +pub enum Event { + Request { labels: Labels }, + Miss { labels: Labels, reason: MissReason }, + Disabled { labels: Labels, reason: DisabledReason }, + Error { labels: Labels, error: ErrorKind, in_fallback: bool }, + Invalidation { namespace: Arc, key_type: Arc, layer: Layer }, + Coalesced { labels: OutcomeLabels, scope: CoalescingScope }, + ShadowValidation { labels: OutcomeLabels, outcome: ShadowOutcome }, + /// Age in seconds of the validated value at a match or confirmed mismatch, clamped at zero. + ShadowValueAge { labels: OutcomeLabels, outcome: ShadowOutcome, seconds: f64 }, + /// Positive offset in seconds of a frame dated after the observing clock. + FutureTimestampOffset { labels: Labels, seconds: f64 }, + StaleRecovery { labels: OutcomeLabels, outcome: RecoveryOutcome }, + StaleRecoveryValueAge { labels: OutcomeLabels, outcome: RecoveryOutcome, seconds: f64 }, + Compression { labels: Labels, outcome: CompressionOutcome }, + Get { labels: Labels, seconds: f64 }, + Fallback { labels: Labels, seconds: f64 }, + Serialization { labels: Labels, operation: SerializationOperation, seconds: f64 }, + Size { labels: Labels, bytes: u64 }, + StoredSize { labels: Labels, bytes: u64 }, + CompressionRatio { labels: Labels, ratio: f64 }, + CompressionDuration { labels: Labels, operation: CompressionOperation, seconds: f64 }, +} + +/// Receives every public diagnostic. Failures (returned errors or panics) +/// never change a cache, source or maintenance result. +pub trait Observer: Send + Sync + 'static { + fn observe(&self, event: &Event); + + /// Whether this observer consumes shadow validation outcomes. + /// + /// Shadow validation is diagnostic work that exists only to be observed, + /// so the cache admits a shadow job only when an observer reports `true`. + fn observes_shadow_outcomes(&self) -> bool { + true + } +} + +/// Bounded details of one confirmed shadow mismatch warning. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ShadowMismatchDetails { + pub namespace: Arc, + pub use_case: Arc, + pub key_type: Arc, + /// The logical key, bounded to 2 KiB of UTF-8. + pub cache_key: String, + /// JSON preview of the cached value, bounded to 8 KiB; absent when unavailable. + pub cached_value_json: Option, + /// JSON preview of the source value, bounded to 8 KiB; absent when unavailable. + pub source_value_json: Option, +} + +/// Severity of a [`LogEvent`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum LogLevel { + Debug, + Warn, + Error, +} + +/// Structured log events. All are fail-open diagnostics except the +/// invalidation failure, which is also returned to the caller. +#[derive(Debug)] +pub enum LogEvent { + KeyConstructionFailed(BoxError), + PolicyResolutionFailed(BoxError), + LocalReadFailed(BoxError), + LocalWriteFailed(BoxError), + RemoteReadFailed(BoxError), + RemoteWriteFailed(BoxError), + RecoveryPredicateFailed(BoxError), + RecoveryDecodeFailed(BoxError), + ShadowFillFailed(BoxError), + ShadowMismatch(ShadowMismatchDetails), + InvalidationFailed(BoxError), +} + +impl LogEvent { + pub fn level(&self) -> LogLevel { + match self { + LogEvent::KeyConstructionFailed(_) | LogEvent::LocalReadFailed(_) => LogLevel::Error, + _ => LogLevel::Warn, + } + } +} + +impl fmt::Display for LogEvent { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + LogEvent::KeyConstructionFailed(e) => write!(f, "Could not construct DialCache key: {e}"), + LogEvent::PolicyResolutionFailed(e) => write!(f, "Could not resolve DialCache key config: {e}"), + LogEvent::LocalReadFailed(e) => write!(f, "Error getting value from local cache: {e}"), + LogEvent::LocalWriteFailed(e) => write!(f, "Error putting value in local cache: {e}"), + LogEvent::RemoteReadFailed(e) => write!(f, "Error getting value from Redis cache: {e}"), + LogEvent::RemoteWriteFailed(e) => write!(f, "Error putting value in Redis cache: {e}"), + LogEvent::RecoveryPredicateFailed(e) => { + write!(f, "DialCache stale recovery predicate failed; recovery was denied: {e}") + } + LogEvent::RecoveryDecodeFailed(e) => write!(f, "Error using retained Redis value during stale recovery: {e}"), + LogEvent::ShadowFillFailed(e) => write!(f, "Error populating Redis from DialCache shadow work: {e}"), + LogEvent::ShadowMismatch(d) => write!( + f, + "DialCache shadow validation mismatch: namespace={} useCase={} keyType={} cacheKey={}", + d.namespace, d.use_case, d.key_type, d.cache_key + ), + LogEvent::InvalidationFailed(e) => write!(f, "Error writing DialCache invalidation watermark: {e}"), + } + } +} + +/// Receives structured log events. Panics are isolated from cache results. +pub trait Logger: Send + Sync + 'static { + fn log(&self, event: &LogEvent); +} + +/// The default logger: the `log` crate facade. +#[derive(Debug, Clone, Copy, Default)] +pub struct LogFacadeLogger; + +impl Logger for LogFacadeLogger { + fn log(&self, event: &LogEvent) { + match event.level() { + LogLevel::Debug => log::debug!(target: "dialcache", "{event}"), + LogLevel::Warn => log::warn!(target: "dialcache", "{event}"), + LogLevel::Error => log::error!(target: "dialcache", "{event}"), + } + } +} diff --git a/rust/src/policy.rs b/rust/src/policy.rs new file mode 100644 index 00000000..3af8fd9a --- /dev/null +++ b/rust/src/policy.rs @@ -0,0 +1,222 @@ +//! Static operation policy, sparse runtime overlays and per-invocation resolution. + +use crate::limits::{DEFAULT_REMOTE_READ_TIMEOUT_MS, MAX_CACHE_TTL_SEC, MAX_DEADLINE_MS}; + +/// Per-use-case shadow validation policy. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct ShadowPolicy { + /// Independent stable cohort percentage in `[0, 100]`. Omitted and zero disable shadow work. + pub ramp: Option, + /// Emit one warning per confirmed mismatch. Defaults to false. + pub log_mismatches: Option, +} + +/// Static caching policy of one operation. Omitted leaves use library +/// defaults or are overridden by a runtime overlay. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct Policy { + /// Memoize values for the outermost enabled scope. Defaults to false. + pub request_local: Option, + /// Share one execution across concurrent same-key callers. Defaults to true. + pub coalesce: Option, + /// Process-local TTL in whole seconds, `1..=MAX_CACHE_TTL_SEC`. Omitted disables the layer. + pub local_ttl_sec: Option, + /// Remote TTL in whole seconds. Omitted disables the layer. + pub remote_ttl_sec: Option, + /// Local serving cohort percentage; a configured TTL implies 100. + pub local_ramp: Option, + /// Remote serving cohort percentage; a configured TTL implies 100. + pub remote_ramp: Option, + /// Exclusive stale-recovery age ceiling in seconds; must exceed the remote TTL. Zero disables. + pub stale_on_error_max_age_sec: Option, + /// Remote read budget in milliseconds, `1..=MAX_DEADLINE_MS`. + pub remote_read_timeout_ms: Option, + pub shadow: Option, +} + +impl Policy { + /// Both serving layers enabled with the same TTL and full ramps. + pub fn enabled(ttl_sec: u64) -> Self { + Policy { + local_ttl_sec: Some(ttl_sec), + remote_ttl_sec: Some(ttl_sec), + local_ramp: Some(100.0), + remote_ramp: Some(100.0), + ..Policy::default() + } + } + + /// The explicit kill switch: request memoization, recovery and shadow + /// work off, both serving layers ramped to zero. As an overlay it disables + /// every inherited path instead of relying on omission. + pub fn disabled() -> Self { + Policy { + request_local: Some(false), + stale_on_error_max_age_sec: Some(0), + local_ramp: Some(0.0), + remote_ramp: Some(0.0), + shadow: Some(ShadowPolicy { ramp: Some(0.0), log_mismatches: Some(false) }), + ..Policy::default() + } + } + + pub fn request_local(mut self, enabled: bool) -> Self { + self.request_local = Some(enabled); + self + } + + pub fn coalesce(mut self, enabled: bool) -> Self { + self.coalesce = Some(enabled); + self + } + + pub fn local_ttl_sec(mut self, ttl_sec: u64) -> Self { + self.local_ttl_sec = Some(ttl_sec); + self + } + + pub fn remote_ttl_sec(mut self, ttl_sec: u64) -> Self { + self.remote_ttl_sec = Some(ttl_sec); + self + } + + pub fn local_ramp(mut self, ramp: f64) -> Self { + self.local_ramp = Some(ramp); + self + } + + pub fn remote_ramp(mut self, ramp: f64) -> Self { + self.remote_ramp = Some(ramp); + self + } + + pub fn stale_on_error_max_age_sec(mut self, seconds: u64) -> Self { + self.stale_on_error_max_age_sec = Some(seconds); + self + } + + pub fn remote_read_timeout_ms(mut self, ms: u64) -> Self { + self.remote_read_timeout_ms = Some(ms); + self + } + + pub fn shadow(mut self, shadow: ShadowPolicy) -> Self { + self.shadow = Some(shadow); + self + } + + /// Parse the JSON-shaped TypeScript configuration + /// (`ttlSec`, `ramp`, `shadow`, `requestLocal`, `coalesce`, + /// `staleOnErrorMaxAgeSec`, `remoteReadTimeoutMs`). `null` or absent means an empty policy. + pub fn from_json(value: &serde_json::Value) -> Result { + let _ = value; + todo!("Policy::from_json") + } + + /// The JSON-shaped form of this policy, with omitted leaves absent. + pub fn to_json(&self) -> serde_json::Value { + todo!("Policy::to_json") + } + + /// Reject statically invalid leaves. + pub fn validate(&self) -> Result<(), PolicyError> { + todo!("Policy::validate") + } +} + +/// A sparse runtime overlay returned by a [`PolicyProvider`](crate::PolicyProvider). +/// +/// Each present leaf replaces the operation leaf; omitted leaves inherit. +/// Leaves may hold invalid values on purpose: their consequences are defined +/// by the portable contract (an invalid TTL or ramp disables only that +/// layer, an invalid flag or read deadline bypasses caching for the call). +#[derive(Debug, Clone, PartialEq)] +pub struct RuntimePolicy(pub serde_json::Value); + +impl RuntimePolicy { + pub fn from_json(value: serde_json::Value) -> Self { + RuntimePolicy(value) + } +} + +impl From for RuntimePolicy { + fn from(policy: Policy) -> Self { + RuntimePolicy(policy.to_json()) + } +} + +impl From for RuntimePolicy { + fn from(value: serde_json::Value) -> Self { + RuntimePolicy(value) + } +} + +/// Instance defaults consulted during resolution. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PolicyDefaults { + pub remote_read_timeout_ms: u64, +} + +impl Default for PolicyDefaults { + fn default() -> Self { + PolicyDefaults { remote_read_timeout_ms: DEFAULT_REMOTE_READ_TIMEOUT_MS } + } +} + +/// Why a whole invocation's policy could not be resolved. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{0}")] +pub struct PolicyError(pub String); + +/// One serving layer after resolution. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct ResolvedLayer { + pub enabled: bool, + /// Why the layer is disabled, when it is. + pub reason: Option, + /// A valid TTL and ramp remain available when the ramp excluded this key. + pub configured: bool, + pub ttl_ms: u64, + pub ramp: f64, +} + +/// Shadow policy after resolution. +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct ResolvedShadow { + /// Cohort selection only; admission still needs an eligible path, hook and capacity. + pub enabled: bool, + pub ramp: f64, + pub log_mismatches: bool, + pub config_error: bool, + /// Recorded only if a job is admitted. + pub logging_config_error: bool, +} + +/// The captured policy of one enabled invocation. +#[derive(Debug, Clone, PartialEq)] +pub struct ResolvedPolicy { + pub request_local: bool, + pub coalesce: bool, + pub local: ResolvedLayer, + pub remote: ResolvedLayer, + pub remote_read_timeout_ms: u64, + /// Zero when recovery is off. + pub stale_on_error_max_age_ms: u64, + pub stale_on_error_config_error: bool, + pub shadow: ResolvedShadow, +} + +/// Merge a static policy with a sparse overlay once, for one logical key. +/// +/// A malformed container, boolean or read deadline fails the whole +/// invocation; TTL or ramp errors disable only that layer; an invalid +/// recovery or shadow option preserves ordinary serving. +pub fn resolve_policy( + base: &Policy, + overlay: Option<&RuntimePolicy>, + logical_key: &str, + defaults: PolicyDefaults, +) -> Result { + let _ = (base, overlay, logical_key, defaults, MAX_CACHE_TTL_SEC, MAX_DEADLINE_MS); + todo!("resolve_policy") +} diff --git a/rust/src/protocol/envelope.rs b/rust/src/protocol/envelope.rs new file mode 100644 index 00000000..e17632f5 --- /dev/null +++ b/rust/src/protocol/envelope.rs @@ -0,0 +1,71 @@ +//! Payload envelopes: escaping and zstd compression. + +use crate::codec::Payload; +use crate::limits::{DEFAULT_COMPRESSION_THRESHOLD_BYTES, DEFAULT_ZSTD_LEVEL, MAX_DECOMPRESSED_BYTES}; +use crate::observe::CompressionOutcome; + +use super::frame::ProtocolError; + +pub const MARKER_ESCAPED_RAW: u8 = 0x00; +pub const MARKER_ZSTD_UTF8: u8 = 0x01; +pub const MARKER_ZSTD_BINARY: u8 = 0x02; + +/// Write-side compression policy. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CompressionConfig { + /// Payloads of at least this many serialized bytes are compressed. + pub threshold_bytes: usize, + /// zstd level, `1..=22`. + pub level: i32, +} + +impl Default for CompressionConfig { + fn default() -> Self { + CompressionConfig { threshold_bytes: DEFAULT_COMPRESSION_THRESHOLD_BYTES, level: DEFAULT_ZSTD_LEVEL } + } +} + +impl CompressionConfig { + pub fn validate(&self) -> Result<(), ProtocolError> { + let _ = MAX_DECOMPRESSED_BYTES; + todo!("CompressionConfig::validate") + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompressionWriteResult { + pub payload: Payload, + pub outcome: CompressionOutcome, + pub original_bytes: usize, + pub stored_bytes: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CompressionReadResult { + pub payload: Payload, + /// `None` when the payload passed through untouched. + pub outcome: Option, +} + +/// Prefix raw binary output whose first byte would collide with the envelope. +pub fn escape_raw_payload(payload: Payload) -> Payload { + let _ = payload; + todo!("escape_raw_payload") +} + +/// Compress when the payload meets the threshold and the marked result is +/// strictly smaller than the escaped raw form. +pub fn compress_payload( + payload: Payload, + config: &CompressionConfig, + max_decompressed_bytes: usize, +) -> Result { + let _ = (payload, config, max_decompressed_bytes); + todo!("compress_payload") +} + +/// Reverse of [`compress_payload`], applied to every read. +pub fn decompress_payload(payload: Payload, max_decompressed_bytes: usize) -> CompressionReadResult { + let _ = (payload, max_decompressed_bytes); + todo!("decompress_payload") +} diff --git a/rust/src/protocol/frame.rs b/rust/src/protocol/frame.rs new file mode 100644 index 00000000..64fec7f0 --- /dev/null +++ b/rust/src/protocol/frame.rs @@ -0,0 +1,67 @@ +//! Version-1 frames and semantic read results. + +use crate::remote::{Frame, ReadResult}; + +/// Wire-level failures that are errors rather than misses. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum ProtocolError { + #[error("Invalid DialCache Redis payload encoding")] + PayloadEncoding, + #[error("Invalid DialCache Redis reply: {0}")] + InvalidReply(String), + #[error("DialCache timestamp must be a nonnegative safe integer")] + InvalidTimestamp, + #[error("DialCache cache TTL must be positive and no greater than 365 days")] + InvalidDuration, + #[error("DialCache compression failed: {0}")] + Compression(String), +} + +/// Encode a frame: version byte, big-endian uint64 stamp, encoding tag, payload. +/// Text payloads receive replacement UTF-8 conversion first. +pub fn encode_frame(frame: &Frame) -> Result, ProtocolError> { + let _ = frame; + todo!("encode_frame") +} + +/// Parse a watermark bulk string. `None` input is absent (a valid zero +/// baseline); `Some(Err)` is malformed; `Some(Ok(value))` is a valid stamp. +pub fn parse_watermark(raw: Option<&[u8]>) -> Option> { + let _ = raw; + todo!("parse_watermark") +} + +/// Classify one raw read with the protocol's precedence: absent value, +/// unsupported frame, tracked marker validity and zero stamp, fence, +/// payload encoding, then a hit. +pub fn decode_frame(raw: Option<&[u8]>, tracked: bool, watermark: Option<&[u8]>) -> Result { + let _ = (raw, tracked, watermark); + todo!("decode_frame") +} + +/// Interpret an untrusted JSON-shaped semantic reply from a custom adapter. +/// Unknown shapes become unclassified misses; frame-shaped objects ignore +/// stray miss metadata. +pub fn read_result_from_untrusted_json(value: &serde_json::Value) -> ReadResult { + let _ = value; + todo!("read_result_from_untrusted_json") +} + +/// The core trust boundary above wire decoding: drop fences on untracked +/// keys or outside the safe domain, and demote a fenced claim without a fence. +pub fn normalize_read_result(result: ReadResult, tracked: bool) -> ReadResult { + let _ = (result, tracked); + todo!("normalize_read_result") +} + +/// A nonnegative safe-integer timestamp. +pub fn validate_timestamp_ms(value: f64) -> Result { + let _ = value; + todo!("validate_timestamp_ms") +} + +/// Round a write TTL up to whole milliseconds and check the 365-day domain. +pub fn ceil_supported_cache_ttl_ms(value: f64) -> Result { + let _ = value; + todo!("ceil_supported_cache_ttl_ms") +} diff --git a/rust/src/protocol/mod.rs b/rust/src/protocol/mod.rs new file mode 100644 index 00000000..5efffea8 --- /dev/null +++ b/rust/src/protocol/mod.rs @@ -0,0 +1,16 @@ +//! The portable wire protocol: frames, envelopes, text conversion and +//! duration/timestamp domains (W04–W08 in `formal/CONTRACTS.md`). + +mod envelope; +mod frame; +mod text; + +pub use envelope::{ + compress_payload, decompress_payload, escape_raw_payload, CompressionConfig, CompressionReadResult, + CompressionWriteResult, MARKER_ESCAPED_RAW, MARKER_ZSTD_BINARY, MARKER_ZSTD_UTF8, +}; +pub use frame::{ + ceil_supported_cache_ttl_ms, decode_frame, encode_frame, normalize_read_result, parse_watermark, + read_result_from_untrusted_json, validate_timestamp_ms, ProtocolError, +}; +pub use text::replacement_utf8; diff --git a/rust/src/protocol/text.rs b/rust/src/protocol/text.rs new file mode 100644 index 00000000..f049f1ef --- /dev/null +++ b/rust/src/protocol/text.rs @@ -0,0 +1,8 @@ +//! Text payload conversion. + +/// Decode bytes as UTF-8 with one U+FFFD per maximal ill-formed subpart and +/// no BOM removal (the WHATWG decoder with replacement error handling). +pub fn replacement_utf8(bytes: &[u8]) -> Vec { + let _ = bytes; + todo!("replacement_utf8") +} diff --git a/rust/src/remote.rs b/rust/src/remote.rs new file mode 100644 index 00000000..01a0d171 --- /dev/null +++ b/rust/src/remote.rs @@ -0,0 +1,126 @@ +//! The remote (Redis) adapter boundary. + +use futures::future::BoxFuture; + +use crate::cancel::CancelToken; +use crate::codec::Payload; +use crate::error::BoxError; + +/// One stored value: the writer's wall-clock stamp and the serialized payload. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Frame { + /// Epoch milliseconds from the writer's clock; a nonnegative safe integer. + pub created_at_ms: u64, + pub payload: Payload, +} + +/// Bounded cause of a semantic miss. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MissReason { + /// The layer had no retrievable value. + ValueAbsent, + /// A valid frame's logical age reached its ceiling. + Expired, + /// A tracked frame was rejected by an invalidation watermark. + WatermarkFenced, + /// A real miss with no decisive cause: malformed frames, unknown replies, + /// invalid or future timestamps, decode failures. + Unclassified, +} + +impl MissReason { + pub fn as_str(self) -> &'static str { + match self { + MissReason::ValueAbsent => "value_absent", + MissReason::Expired => "expired", + MissReason::WatermarkFenced => "watermark_fenced", + MissReason::Unclassified => "unclassified", + } + } + + pub fn parse(text: &str) -> Option { + Some(match text { + "value_absent" => MissReason::ValueAbsent, + "expired" => MissReason::Expired, + "watermark_fenced" => MissReason::WatermarkFenced, + "unclassified" => MissReason::Unclassified, + _ => return None, + }) + } +} + +/// Semantic result of one remote read. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ReadResult { + /// A complete frame that passed decoding and, for tracked keys, the fence. + Hit(Frame), + /// A classified miss. `observed_watermark_ms` is the valid invalidation + /// watermark read atomically with a tracked value, when one existed; a + /// refill stamped at or before it is known to remain unreadable. + Miss { + reason: MissReason, + observed_watermark_ms: Option, + }, +} + +impl ReadResult { + pub fn miss(reason: MissReason) -> Self { + ReadResult::Miss { reason, observed_watermark_ms: None } + } + + pub fn is_miss(&self) -> bool { + matches!(self, ReadResult::Miss { .. }) + } +} + +/// Keys of one read: the value key and, for tracked identities, the watermark key. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ReadRequest { + pub value_key: String, + pub watermark_key: Option, +} + +/// The effective read budget and a cooperative cancellation request. +/// +/// The cache's own deadline remains authoritative; adapters may use the token +/// to stop waiting, but a dispatched command may still execute. +#[derive(Debug, Clone)] +pub struct ReadContext { + pub timeout_ms: u64, + pub cancel: CancelToken, +} + +/// One complete-frame write: a single native `SET value_key frame PX ttl_ms`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WriteRequest { + pub value_key: String, + pub frame: Frame, + /// Positive milliseconds no greater than 365 days. + pub ttl_ms: u64, +} + +/// One invalidation: advance the entity watermark monotonically. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InvalidateRequest { + pub watermark_key: String, + /// The invalidating process's wall clock sample; reused through retries. + pub invalidated_at_ms: u64, + /// Nonnegative milliseconds no greater than 365 days. + pub future_buffer_ms: u64, +} + +/// Caller-owned semantic Redis boundary. +/// +/// The cache borrows the adapter and never connects, drains or closes it. +/// Tracked reads must observe the value and watermark atomically from an +/// authoritative primary. Writes are one native `SET` of a complete frame +/// stamped by the cache; they never create or extend watermarks. +/// Invalidation runs the shared Lua transition. Use +/// [`crate::protocol::decode_frame`] and [`crate::protocol::encode_frame`] +/// for the wire format. +pub trait Remote: Send + Sync + 'static { + fn read(&self, request: ReadRequest, context: ReadContext) -> BoxFuture<'_, Result>; + fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>>; + fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>>; +} diff --git a/rust/src/spawn.rs b/rust/src/spawn.rs new file mode 100644 index 00000000..b1310ed1 --- /dev/null +++ b/rust/src/spawn.rs @@ -0,0 +1,29 @@ +//! Detached task admission. + +use futures::future::BoxFuture; + +/// Runs detached work the cache does not await: raw source or adapter work +/// that outlives a caller's deadline, and shadow validation jobs. +pub trait Spawner: Send + Sync + 'static { + /// Start `task` concurrently with the caller. + fn spawn(&self, task: BoxFuture<'static, ()>); + /// Start `task` after work that is already runnable has progressed. + /// + /// Production runtimes treat this like [`Spawner::spawn`]. A controlled + /// test executor runs deferred work only once everything else is blocked. + fn defer(&self, task: BoxFuture<'static, ()>) { + self.spawn(task) + } +} + +/// Spawns onto the ambient tokio runtime. +#[cfg(feature = "tokio")] +#[derive(Debug, Clone, Copy, Default)] +pub struct TokioSpawner; + +#[cfg(feature = "tokio")] +impl Spawner for TokioSpawner { + fn spawn(&self, task: BoxFuture<'static, ()>) { + tokio::spawn(task); + } +} diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs new file mode 100644 index 00000000..b2f5dc52 --- /dev/null +++ b/rust/tests/conformance.rs @@ -0,0 +1,3 @@ +fn main() { + println!("conformance harness placeholder"); +} From 2880483441c142fffeaf3d677031aaf0e6cf82fb Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 00:21:01 -0700 Subject: [PATCH 02/31] feat(rust): implement identity keys, argument normalization and rollout cohorts with protocol vector replay --- rust/src/identity.rs | 340 ++++++++++++++++++++++++-- rust/tests/formal/fixtures.rs | 401 +++++++++++++++++++++++++++++++ rust/tests/formal/key_vectors.rs | 238 ++++++++++++++++++ rust/tests/protocol_keys.rs | 51 ++++ 4 files changed, 1016 insertions(+), 14 deletions(-) create mode 100644 rust/tests/formal/fixtures.rs create mode 100644 rust/tests/formal/key_vectors.rs create mode 100644 rust/tests/protocol_keys.rs diff --git a/rust/src/identity.rs b/rust/src/identity.rs index f232b9fb..eb9546c0 100644 --- a/rust/src/identity.rs +++ b/rust/src/identity.rs @@ -4,6 +4,11 @@ //! escaping, tracked entity hash tags, ordered argument pairs, the frame key //! suffix, and FNV-1a cohorts over UTF-16 code units. +use std::cmp::Ordering; +use std::fmt::Write as _; + +use crate::limits::FRAME_KEY_SUFFIX; + /// A normalized logical identity. Ordered arguments retain caller order; /// use [`normalize_args`] to build them from a host-language record. #[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] @@ -22,7 +27,11 @@ pub struct Identity { } impl Identity { - pub fn new(key_type: impl Into, id: impl Into, use_case: impl Into) -> Self { + pub fn new( + key_type: impl Into, + id: impl Into, + use_case: impl Into, + ) -> Self { Identity { namespace: String::new(), key_type: key_type.into(), @@ -90,44 +99,347 @@ pub enum ArgValue { Str(String), } +impl ArgValue { + /// The JavaScript `String(value)` spelling, or `None` for an absent value. + fn spell(self) -> Result, IdentityError> { + Ok(Some(match self { + ArgValue::Absent => return Ok(None), + ArgValue::Null => "null".to_owned(), + ArgValue::Bool(value) => if value { "true" } else { "false" }.to_owned(), + ArgValue::Number(value) => js_number_to_string(value), + ArgValue::Int(value) => value.to_string(), + ArgValue::BigInt(text) => { + if !is_decimal_integer(&text) { + return Err(IdentityError::UnsupportedArgument(format!( + "bigint text {text:?} is not a decimal integer" + ))); + } + text + } + ArgValue::Str(text) => text, + })) + } +} + +fn is_decimal_integer(text: &str) -> bool { + let digits = text.strip_prefix('-').unwrap_or(text); + !digits.is_empty() && digits.bytes().all(|byte| byte.is_ascii_digit()) +} + /// Normalize a record of key arguments: omit absent values, spell scalars /// the JavaScript way, and sort names by UTF-16 code units. +/// +/// The sort is stable, so entries whose names compare equal keep their input order. pub fn normalize_args(args: I) -> Result, IdentityError> where I: IntoIterator, K: Into, { - let _ = args; - todo!("normalize_args") + let mut pairs = Vec::new(); + for (name, value) in args { + if let Some(text) = value.spell()? { + pairs.push((name.into(), text)); + } + } + pairs.sort_by(|left, right| compare_utf16(&left.0, &right.0)); + Ok(pairs) +} + +fn contains_brace(component: &str) -> bool { + component.contains(['{', '}']) } /// Build every key of an identity. +/// +/// The namespace may never contain a hash-tag brace. The key type and ID +/// are restricted the same way only when the identity is tracked, because +/// only then do they form the Redis Cluster hash tag; untracked braces are +/// percent-escaped like any other reserved character. pub fn keys(identity: &Identity) -> Result { - let _ = identity; - todo!("keys") + if contains_brace(&identity.namespace) + || (identity.tracked + && (contains_brace(&identity.key_type) || contains_brace(&identity.id))) + { + return Err(IdentityError::ReservedDelimiter); + } + let entity = format!( + "{}:{}:{}", + escape_component(&identity.namespace), + escape_component(&identity.key_type), + escape_component(&identity.id) + ); + let (mut logical, watermark) = if identity.tracked { + let tagged = format!("{{{entity}}}"); + let watermark = format!("{tagged}#watermark"); + (tagged, Some(watermark)) + } else { + (entity, None) + }; + for (index, (name, value)) in identity.args.iter().enumerate() { + logical.push(if index == 0 { '?' } else { '&' }); + logical.push_str(&escape_component(name)); + logical.push('='); + logical.push_str(&escape_component(value)); + } + logical.push('#'); + logical.push_str(&escape_component(&identity.use_case)); + let value = format!("{logical}{FRAME_KEY_SUFFIX}"); + Ok(Keys { + logical, + value, + watermark, + }) } /// `encodeURIComponent` escaping of one component. +/// +/// Bytes of the UTF-8 encoding are kept literally when they are +/// `A-Z a-z 0-9 - _ . ! ~ * ' ( )` and percent-encoded with uppercase hex otherwise. pub fn escape_component(component: &str) -> String { - let _ = component; - todo!("escape_component") + let mut escaped = String::with_capacity(component.len()); + for byte in component.bytes() { + if byte.is_ascii_alphanumeric() || b"-_.!~*'()".contains(&byte) { + escaped.push(byte as char); + } else { + let _ = write!(escaped, "%{byte:02X}"); + } + } + escaped +} + +/// FNV-1a 32-bit hash over the UTF-16 code units of `logical_key:discriminator`; +/// the integer numerator behind [`cohort`]. +pub fn cohort_hash(logical_key: &str, discriminator: &str) -> u32 { + let units = logical_key + .encode_utf16() + .chain(":".encode_utf16()) + .chain(discriminator.encode_utf16()); + units.fold(0x811c_9dc5_u32, |hash, unit| { + (hash ^ u32::from(unit)).wrapping_mul(0x0100_0193) + }) } /// Deterministic rollout sample in `[0, 100)` for a logical key and layer or /// shadow discriminator: FNV-1a over the UTF-16 units of `key:discriminator`. pub fn cohort(logical_key: &str, discriminator: &str) -> f64 { - let _ = (logical_key, discriminator); - todo!("cohort") + f64::from(cohort_hash(logical_key, discriminator)) / 4_294_967_296.0 * 100.0 } /// Spell a number the way JavaScript's `String(number)` does. +/// +/// This is ECMA-262 `Number::toString(10)`: the shortest digit string that +/// round-trips, laid out as plain decimal for exponents in `(-7, 21]` and as +/// `d.ddde±X` otherwise. Negative zero spells `0`. pub fn js_number_to_string(value: f64) -> String { - let _ = value; - todo!("js_number_to_string") + if value.is_nan() { + return "NaN".to_owned(); + } + if value.is_infinite() { + return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned(); + } + if value == 0.0 { + return "0".to_owned(); + } + let (digits, exponent) = shortest_digits(value.abs()); + let k = digits.len() as i32; + let n = exponent + 1; + let mut out = String::new(); + if value < 0.0 { + out.push('-'); + } + if k <= n && n <= 21 { + out.push_str(&digits); + out.extend(std::iter::repeat_n('0', (n - k) as usize)); + } else if 0 < n && n <= 21 { + let (whole, fraction) = digits.split_at(n as usize); + out.push_str(whole); + out.push('.'); + out.push_str(fraction); + } else if -6 < n && n <= 0 { + out.push_str("0."); + out.extend(std::iter::repeat_n('0', (-n) as usize)); + out.push_str(&digits); + } else { + let (first, rest) = digits.split_at(1); + out.push_str(first); + if !rest.is_empty() { + out.push('.'); + out.push_str(rest); + } + let e = n - 1; + out.push('e'); + out.push(if e < 0 { '-' } else { '+' }); + let _ = write!(out, "{}", e.abs()); + } + out +} + +/// The shortest round-trip decimal digits of a positive finite number and +/// the decimal exponent of its first digit: `value = 0.d1d2… × 10^(exponent+1)`. +fn shortest_digits(value: f64) -> (String, i32) { + debug_assert!(value.is_finite() && value > 0.0); + // Rust's `LowerExp` without a precision prints the shortest representation + // that round-trips, as `d[.ddd]e[-]X`. + let scientific = format!("{value:e}"); + let (mantissa, exponent) = scientific + .split_once('e') + .expect("LowerExp always includes an exponent"); + let mut digits: String = mantissa.chars().filter(|c| *c != '.').collect(); + let trimmed = digits.trim_end_matches('0').len().max(1); + digits.truncate(trimmed); + ( + digits, + exponent + .parse() + .expect("LowerExp exponent is a decimal integer"), + ) } /// Compare two strings by UTF-16 code units, as JavaScript's `<` does. -pub fn compare_utf16(left: &str, right: &str) -> std::cmp::Ordering { - let _ = (left, right); - todo!("compare_utf16") +pub fn compare_utf16(left: &str, right: &str) -> Ordering { + left.encode_utf16().cmp(right.encode_utf16()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn js_numbers_follow_ecmascript_spelling() { + let cases: &[(f64, &str)] = &[ + (f64::NAN, "NaN"), + (f64::INFINITY, "Infinity"), + (f64::NEG_INFINITY, "-Infinity"), + (0.0, "0"), + (-0.0, "0"), + (1.5, "1.5"), + (-1.5, "-1.5"), + (100.0, "100"), + (1e21, "1e+21"), + (1e-7, "1e-7"), + (-1e-7, "-1e-7"), + (0.000001, "0.000001"), + (123456789012345680000.0, "123456789012345680000"), + (9007199254740991.0, "9007199254740991"), + (1.25, "1.25"), + (0.1, "0.1"), + (1.2345e-7, "1.2345e-7"), + (1.5e300, "1.5e+300"), + (5e-324, "5e-324"), + (f64::MAX, "1.7976931348623157e+308"), + (0.5, "0.5"), + (123.456, "123.456"), + (1e20, "100000000000000000000"), + ]; + for (value, expected) in cases { + assert_eq!( + js_number_to_string(*value), + *expected, + "spelling of {value:?}" + ); + } + } + + #[test] + fn escaping_matches_encode_uri_component() { + assert_eq!(escape_component("é"), "%C3%A9"); + assert_eq!(escape_component("😀"), "%F0%9F%98%80"); + assert_eq!(escape_component("~!*'()-._"), "~!*'()-._"); + assert_eq!( + escape_component("a b+c/d?e=f&g#h%"), + "a%20b%2Bc%2Fd%3Fe%3Df%26g%23h%25" + ); + assert_eq!(escape_component("{1}"), "%7B1%7D"); + assert_eq!(escape_component(""), ""); + } + + #[test] + fn utf16_ordering_places_astral_before_private_use() { + assert_eq!(compare_utf16("😀", "\u{e000}"), Ordering::Less); + assert_eq!(compare_utf16("\u{ffff}", "😀"), Ordering::Greater); + assert_eq!(compare_utf16("a", "ab"), Ordering::Less); + assert_eq!(compare_utf16("b", "ab"), Ordering::Greater); + assert_eq!(compare_utf16("", ""), Ordering::Equal); + } + + #[test] + fn normalization_spells_scalars_and_sorts_names() { + let normalized = normalize_args(vec![ + ("z", ArgValue::Str("last".into())), + ("missing", ArgValue::Absent), + ("a", ArgValue::Int(1)), + ("nil", ArgValue::Null), + ("flag", ArgValue::Bool(false)), + ("wide", ArgValue::BigInt("-9007199254740993".into())), + ("tiny", ArgValue::Number(1e-7)), + ]) + .unwrap(); + let expected: Vec<(String, String)> = [ + ("a", "1"), + ("flag", "false"), + ("nil", "null"), + ("tiny", "1e-7"), + ("wide", "-9007199254740993"), + ("z", "last"), + ] + .into_iter() + .map(|(name, value)| (name.to_owned(), value.to_owned())) + .collect(); + assert_eq!(normalized, expected); + } + + #[test] + fn malformed_bigint_text_is_rejected() { + for text in ["", "-", "1.5", "0x10", "1e3", "+1", "１"] { + assert!( + matches!( + normalize_args(vec![("n", ArgValue::BigInt(text.into()))]), + Err(IdentityError::UnsupportedArgument(_)) + ), + "bigint text {text:?} should be rejected" + ); + } + } + + #[test] + fn braces_are_reserved_only_where_they_form_a_hash_tag() { + let untracked = Identity::new("{kind}", "{1}", "Get").namespace("urn"); + assert_eq!( + untracked.keys().unwrap().logical, + "urn:%7Bkind%7D:%7B1%7D#Get" + ); + assert_eq!( + untracked.clone().tracked(true).keys(), + Err(IdentityError::ReservedDelimiter) + ); + assert_eq!( + Identity::new("k", "1", "Get").namespace("bad{ns").keys(), + Err(IdentityError::ReservedDelimiter) + ); + let tracked = Identity::new("user_id", "123", "GetUser") + .namespace("users-api") + .tracked(true) + .args(vec![("locale".into(), "en".into())]); + let keys = tracked.keys().unwrap(); + assert_eq!(keys.logical, "{users-api:user_id:123}?locale=en#GetUser"); + assert_eq!( + keys.value, + "{users-api:user_id:123}?locale=en#GetUser:dialcache-frame-v1" + ); + assert_eq!( + keys.watermark.as_deref(), + Some("{users-api:user_id:123}#watermark") + ); + } + + #[test] + fn cohort_is_fnv1a_over_utf16_units() { + assert_eq!(cohort_hash("urn:id:1#Get", "local"), 1_652_509_740); + assert_eq!(cohort("urn:id:1#Get", "local"), 38.475490640848875); + assert_eq!(cohort_hash("", ""), { + let mut hash = 0x811c_9dc5_u32; + hash ^= u32::from(b':'); + hash.wrapping_mul(0x0100_0193) + }); + } } diff --git a/rust/tests/formal/fixtures.rs b/rust/tests/formal/fixtures.rs new file mode 100644 index 00000000..9489981d --- /dev/null +++ b/rust/tests/formal/fixtures.rs @@ -0,0 +1,401 @@ +//! Shared loading of the portable protocol vector corpus. +//! +//! Mirrors `go/protocol_test.go` `vectors()` and +//! `go/generated_protocol_test.go`: the fixed vectors in +//! `formal/protocol-vectors.json` are merged with every generated protocol +//! artifact registered in `formal/execution.json`, after the artifact's +//! provenance fingerprints are verified against the model sources. +//! +//! Include from a test binary with `#[path = "formal/fixtures.rs"] mod fixtures;`. + +#![allow(dead_code)] + +use std::collections::BTreeMap; +use std::fs; +use std::path::{Path, PathBuf}; + +use serde_json::Value; + +/// Replaces every lone UTF-16 surrogate escape in fixture JSON so the text +/// can be parsed at all. Components carrying this marker are unrepresentable +/// as Rust strings and are rejected at the fixture boundary. +pub const LONE_SURROGATE_MARKER: char = '\u{E000}'; + +/// Environment variable selecting the corpus: `all` (default), `generated` or `fixed`. +pub const CORPUS_ENV: &str = "DIALCACHE_PROTOCOL_CORPUS"; + +/// Resolve a repository-relative path from the crate manifest directory's parent. +pub fn repo_path(relative: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(relative) +} + +fn read_repo_text(relative: &str) -> String { + let path = repo_path(relative); + fs::read_to_string(&path).unwrap_or_else(|error| panic!("read {}: {error}", path.display())) +} + +/// Parse fixture JSON that may contain lone surrogate escapes (`\ud800`, +/// `\udc00`), which `serde_json` rejects. Each lone escape is replaced with +/// [`LONE_SURROGATE_MARKER`] before parsing; valid surrogate pairs and every +/// other escape are left untouched. +pub fn load_json_marking_lone_surrogates(path: impl AsRef) -> Value { + let path = path.as_ref(); + let text = + fs::read_to_string(path).unwrap_or_else(|error| panic!("read {}: {error}", path.display())); + let marked = mark_lone_surrogate_escapes(&text); + serde_json::from_str(&marked) + .unwrap_or_else(|error| panic!("parse {}: {error}", path.display())) +} + +/// Replace every lone surrogate escape in JSON source text with `\ue000`. +pub fn mark_lone_surrogate_escapes(text: &str) -> String { + const MARKER_ESCAPE: &str = "\\ue000"; + let bytes = text.as_bytes(); + let mut out = String::with_capacity(text.len()); + let mut index = 0; + while index < bytes.len() { + if bytes[index] != b'\\' { + let rest = &text[index..]; + let ch = rest.chars().next().expect("index is on a char boundary"); + out.push(ch); + index += ch.len_utf8(); + continue; + } + match unicode_escape_at(bytes, index) { + Some(unit) if (0xD800..=0xDBFF).contains(&unit) => { + let pair = unicode_escape_at(bytes, index + 6) + .filter(|low| (0xDC00..=0xDFFF).contains(low)); + if pair.is_some() { + out.push_str(&text[index..index + 12]); + index += 12; + } else { + out.push_str(MARKER_ESCAPE); + index += 6; + } + } + Some(unit) if (0xDC00..=0xDFFF).contains(&unit) => { + // A low surrogate preceded by a high one was consumed with its pair above. + out.push_str(MARKER_ESCAPE); + index += 6; + } + Some(_) => { + out.push_str(&text[index..index + 6]); + index += 6; + } + None => { + // Any other escape: copy the backslash and the escaped byte so a + // `\\` cannot be mistaken for the start of a `\u` escape. + let end = (index + 2).min(bytes.len()); + out.push_str(&text[index..end]); + index = end; + } + } + } + out +} + +/// The code unit of a `\uXXXX` escape starting at `index`, when one is there. +fn unicode_escape_at(bytes: &[u8], index: usize) -> Option { + if index + 6 > bytes.len() || bytes[index] != b'\\' || bytes[index + 1] != b'u' { + return None; + } + let hex = std::str::from_utf8(&bytes[index + 2..index + 6]).ok()?; + if !hex.bytes().all(|byte| byte.is_ascii_hexdigit()) { + return None; + } + u16::from_str_radix(hex, 16).ok() +} + +/// Hex-encoded SHA-256 of `bytes`. +pub fn sha256_hex(bytes: &[u8]) -> String { + sha256(bytes) + .iter() + .fold(String::with_capacity(64), |mut hex, byte| { + use std::fmt::Write as _; + let _ = write!(hex, "{byte:02x}"); + hex + }) +} + +/// SHA-256 (FIPS 180-4) without a dependency; the fixture reader only needs +/// to fingerprint a handful of small source files. +pub fn sha256(message: &[u8]) -> [u8; 32] { + const K: [u32; 64] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, + 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, + 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, + 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, + 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, + 0xc67178f2, + ]; + let mut state: [u32; 8] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, + 0x5be0cd19, + ]; + + let mut padded = message.to_vec(); + padded.push(0x80); + while padded.len() % 64 != 56 { + padded.push(0); + } + padded.extend_from_slice(&((message.len() as u64) * 8).to_be_bytes()); + + for block in padded.chunks_exact(64) { + let mut w = [0u32; 64]; + for (i, word) in block.chunks_exact(4).enumerate() { + w[i] = u32::from_be_bytes([word[0], word[1], word[2], word[3]]); + } + for i in 16..64 { + let s0 = w[i - 15].rotate_right(7) ^ w[i - 15].rotate_right(18) ^ (w[i - 15] >> 3); + let s1 = w[i - 2].rotate_right(17) ^ w[i - 2].rotate_right(19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + .wrapping_add(s0) + .wrapping_add(w[i - 7]) + .wrapping_add(s1); + } + let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut h] = state; + for i in 0..64 { + let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); + let ch = (e & f) ^ (!e & g); + let t1 = h + .wrapping_add(s1) + .wrapping_add(ch) + .wrapping_add(K[i]) + .wrapping_add(w[i]); + let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); + let maj = (a & b) ^ (a & c) ^ (b & c); + let t2 = s0.wrapping_add(maj); + h = g; + g = f; + f = e; + e = d.wrapping_add(t1); + d = c; + c = b; + b = a; + a = t1.wrapping_add(t2); + } + for (slot, value) in state.iter_mut().zip([a, b, c, d, e, f, g, h]) { + *slot = slot.wrapping_add(value); + } + } + let mut digest = [0u8; 32]; + for (chunk, word) in digest.chunks_exact_mut(4).zip(state) { + chunk.copy_from_slice(&word.to_be_bytes()); + } + digest +} + +/// Every generated protocol vector group registered in `formal/execution.json`, +/// keyed by field name, after validating each artifact's schema, provenance +/// and case inventory the way `go/generated_protocol_test.go` does. +pub fn quint_protocol_groups() -> BTreeMap> { + let manifest: Value = serde_json::from_str(&read_repo_text("formal/execution.json")) + .expect("parse formal/execution.json"); + let models = manifest["models"] + .as_array() + .expect("execution.json models array"); + let mut groups: BTreeMap> = BTreeMap::new(); + for model in models { + let Some(export) = model + .get("vectorExport") + .filter(|export| export["kind"] == "protocol") + else { + continue; + }; + let model_path = model["path"].as_str().expect("model path"); + let artifact = export["artifact"].as_str().expect("vectorExport.artifact"); + let sources: Vec<&str> = export["sources"] + .as_array() + .expect("vectorExport.sources") + .iter() + .map(|s| s.as_str().expect("source path")) + .collect(); + let cases = export["cases"].as_u64().expect("vectorExport.cases") as usize; + + let envelope = load_json_marking_lone_surrogates(repo_path(artifact)); + let object = envelope + .as_object() + .unwrap_or_else(|| panic!("{artifact}: artifact is not an object")); + assert_eq!( + object.get("schemaVersion"), + Some(&Value::from(3)), + "{artifact}: unsupported schemaVersion" + ); + let provenance = &object["provenance"]; + assert_eq!( + provenance["model"].as_str(), + Some(model_path), + "{artifact}: provenance.model mismatch" + ); + let digests = provenance["sourceSha256"] + .as_object() + .unwrap_or_else(|| panic!("{artifact}: provenance.sourceSha256 missing")); + assert_eq!( + digests.len(), + sources.len(), + "{artifact}: provenance source count differs from execution.json" + ); + for source in &sources { + let bytes = fs::read(repo_path(source)) + .unwrap_or_else(|error| panic!("read {source}: {error}")); + let expected = digests + .get(*source) + .and_then(Value::as_str) + .unwrap_or_else(|| panic!("{artifact}: no provenance digest for {source}")); + assert_eq!( + sha256_hex(&bytes), + expected, + "stale generated protocol source: {source}" + ); + } + + let mut seen = std::collections::BTreeSet::new(); + let mut count = 0; + for (field, rows) in object { + if field == "schemaVersion" || field == "provenance" { + continue; + } + let rows = rows + .as_array() + .unwrap_or_else(|| panic!("{artifact}: field {field} is not an array")); + for row in rows { + let name = row["name"].as_str().unwrap_or(""); + assert!( + !name.is_empty() && seen.insert(name.to_owned()), + "{artifact}: duplicate or missing generated protocol case {name:?}" + ); + } + count += rows.len(); + groups + .entry(field.clone()) + .or_default() + .extend(rows.iter().cloned()); + } + assert_eq!( + count, cases, + "{artifact}: incomplete generated protocol vector inventory" + ); + } + groups +} + +/// The effective corpus selection: the `DIALCACHE_PROTOCOL_CORPUS` variable +/// when set and non-empty, otherwise `default`. +pub fn corpus_selection(default: &str) -> String { + match std::env::var(CORPUS_ENV) { + Ok(value) if !value.is_empty() => value, + _ => default.to_owned(), + } +} + +/// The fixed protocol vectors merged with the generated groups, keyed by +/// group name. `selection` is the default corpus (`all`, `generated` or +/// `fixed`); `DIALCACHE_PROTOCOL_CORPUS` overrides it like `go/protocol_test.go`. +pub fn protocol_groups(selection: &str) -> BTreeMap> { + let selection = corpus_selection(selection); + assert!( + matches!(selection.as_str(), "all" | "generated" | "fixed"), + "unknown protocol corpus selection {selection:?}" + ); + + let fixed = load_json_marking_lone_surrogates(repo_path("formal/protocol-vectors.json")); + let fixed = fixed + .as_object() + .expect("protocol-vectors.json is an object"); + assert_eq!( + fixed.get("schemaVersion"), + Some(&Value::from(3)), + "unsupported protocol vector schema" + ); + let mut groups: BTreeMap> = fixed + .iter() + .filter(|(name, _)| name.as_str() != "schemaVersion") + .map(|(name, rows)| { + let rows = rows + .as_array() + .unwrap_or_else(|| panic!("protocol-vectors.json: {name} is not an array")); + ( + name.clone(), + if selection == "generated" { + Vec::new() + } else { + rows.clone() + }, + ) + }) + .collect(); + if selection != "fixed" { + for (name, rows) in quint_protocol_groups() { + groups + .get_mut(&name) + .unwrap_or_else(|| panic!("unknown generated protocol group {name}")) + .extend(rows); + } + } + groups +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sha256_matches_known_digests() { + assert_eq!( + sha256_hex(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + sha256_hex(b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + let long = b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"; + assert_eq!( + sha256_hex(long), + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + ); + assert_eq!( + sha256_hex(&[b'a'; 1_000_000]), + "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0" + ); + } + + #[test] + fn lone_surrogates_are_marked_and_pairs_kept() { + assert_eq!(mark_lone_surrogate_escapes(r#""\ud800""#), r#""\ue000""#); + assert_eq!(mark_lone_surrogate_escapes(r#""\udc00""#), r#""\ue000""#); + assert_eq!( + mark_lone_surrogate_escapes(r#""\ud83d\ude00""#), + r#""\ud83d\ude00""# + ); + assert_eq!( + mark_lone_surrogate_escapes(r#""\udc00\ud800""#), + r#""\ue000\ue000""# + ); + assert_eq!( + mark_lone_surrogate_escapes(r#""\ud800\ud800\udc00""#), + r#""\ue000\ud800\udc00""# + ); + assert_eq!( + mark_lone_surrogate_escapes(r#""a\\ud800""#), + r#""a\\ud800""# + ); + assert_eq!( + mark_lone_surrogate_escapes(r#""\u00e9\ue000""#), + r#""\u00e9\ue000""# + ); + assert_eq!( + mark_lone_surrogate_escapes(r#""\uD800x\uDC00""#), + r#""\ue000x\ue000""# + ); + let parsed: Value = + serde_json::from_str(&mark_lone_surrogate_escapes(r#"{"id":"x\ud800y"}"#)).unwrap(); + assert_eq!(parsed["id"].as_str(), Some("x\u{e000}y")); + } +} diff --git a/rust/tests/formal/key_vectors.rs b/rust/tests/formal/key_vectors.rs new file mode 100644 index 00000000..744ad44d --- /dev/null +++ b/rust/tests/formal/key_vectors.rs @@ -0,0 +1,238 @@ +//! Replay of the key, invalid-key, argument-normalization and rollout vector +//! groups against `dialcache::identity`. +//! +//! Include from a test binary alongside `fixtures.rs`: +//! `#[path = "formal/fixtures.rs"] mod fixtures;` +//! `#[path = "formal/key_vectors.rs"] mod key_vectors;` + +#![allow(dead_code)] + +use std::collections::BTreeMap; + +use dialcache::identity::{cohort, cohort_hash, normalize_args, ArgValue, Identity}; +use serde_json::Value; + +use super::fixtures::LONE_SURROGATE_MARKER; + +fn identity_from(value: &Value) -> Result { + serde_json::from_value(value.clone()) + .map_err(|error| format!("identity input does not parse: {error}")) +} + +fn components(identity: &Identity) -> impl Iterator { + [ + identity.namespace.as_str(), + identity.key_type.as_str(), + identity.id.as_str(), + identity.use_case.as_str(), + ] + .into_iter() + .chain( + identity + .args + .iter() + .flat_map(|(name, value)| [name.as_str(), value.as_str()]), + ) +} + +fn expected_str(value: &Value, field: &str) -> Result, String> { + match value.get(field) { + None | Some(Value::Null) => Ok(None), + Some(Value::String(text)) => Ok(Some(text.clone())), + Some(other) => Err(format!("{field} is neither a string nor null: {other}")), + } +} + +/// `keyVectors`: `keys()` must succeed and equal `logicalKey`, `valueKey` and `watermarkKey`. +pub fn check_key_vector(vector: &Value) -> Result<(), String> { + let identity = identity_from(&vector["input"])?; + let keys = identity + .keys() + .map_err(|error| format!("keys rejected a valid identity: {error}"))?; + let logical = expected_str(vector, "logicalKey")?.ok_or("logicalKey missing")?; + let value = expected_str(vector, "valueKey")?.ok_or("valueKey missing")?; + let watermark = expected_str(vector, "watermarkKey")?; + if keys.logical != logical { + return Err(format!( + "logical key {:?} != expected {logical:?}", + keys.logical + )); + } + if keys.value != value { + return Err(format!("value key {:?} != expected {value:?}", keys.value)); + } + if keys.watermark != watermark { + return Err(format!( + "watermark key {:?} != expected {watermark:?}", + keys.watermark + )); + } + Ok(()) +} + +/// Decode one `inputUtf16` field: `Ok(None)` when the units are not valid UTF-16. +fn utf16_field(value: &Value, what: &str) -> Result, String> { + let units = value + .as_array() + .ok_or_else(|| format!("inputUtf16 {what} is not an array"))?; + let units: Vec = units + .iter() + .map(|unit| { + unit.as_u64() + .and_then(|unit| u16::try_from(unit).ok()) + .ok_or_else(|| format!("inputUtf16 {what} has an invalid unit {unit}")) + }) + .collect::>()?; + Ok(String::from_utf16(&units).ok()) +} + +/// Rebuild the identity from `inputUtf16` code units, keeping the tracking +/// flag of the parsed input; `Ok(None)` when any field is not valid UTF-16. +fn identity_from_utf16(raw: &Value, tracked: bool) -> Result, String> { + let Some(namespace) = utf16_field(&raw["namespace"], "namespace")? else { + return Ok(None); + }; + let Some(key_type) = utf16_field(&raw["keyType"], "keyType")? else { + return Ok(None); + }; + let Some(id) = utf16_field(&raw["id"], "id")? else { + return Ok(None); + }; + let Some(use_case) = utf16_field(&raw["useCase"], "useCase")? else { + return Ok(None); + }; + let mut args = Vec::new(); + for pair in raw["args"] + .as_array() + .ok_or("inputUtf16 args is not an array")? + { + let pair = pair + .as_array() + .filter(|pair| pair.len() == 2) + .ok_or("inputUtf16 args entry is not a pair")?; + let Some(name) = utf16_field(&pair[0], "argument name")? else { + return Ok(None); + }; + let Some(value) = utf16_field(&pair[1], "argument value")? else { + return Ok(None); + }; + args.push((name, value)); + } + Ok(Some(Identity { + namespace, + key_type, + id, + use_case, + tracked, + args, + })) +} + +/// `invalidKeyVectors`: the identity must be rejected. A component carrying +/// [`LONE_SURROGATE_MARKER`] was already rejected while decoding the fixture, +/// which the protocol allows for scalar-only string types; every other input +/// must make `keys()` fail. When `inputUtf16` is present the identity is also +/// rebuilt from code units and must fail either UTF-16 decoding or `keys()`. +pub fn check_invalid_key_vector(vector: &Value) -> Result<(), String> { + let identity = identity_from(&vector["input"])?; + let unrepresentable = + components(&identity).any(|component| component.contains(LONE_SURROGATE_MARKER)); + if !unrepresentable && identity.keys().is_ok() { + return Err("invalid key accepted".to_owned()); + } + if let Some(raw) = vector.get("inputUtf16") { + if let Some(rebuilt) = identity_from_utf16(raw, identity.tracked)? { + if rebuilt.keys().is_ok() { + return Err("invalid key rebuilt from UTF-16 units accepted".to_owned()); + } + } + } + Ok(()) +} + +fn arg_from_json(value: &Value) -> Result { + Ok(match value { + Value::Null => ArgValue::Null, + Value::Bool(flag) => ArgValue::Bool(*flag), + Value::Number(number) => match number.as_i64() { + Some(integer) => ArgValue::Int(integer), + None => ArgValue::Number( + number + .as_f64() + .ok_or_else(|| format!("number {number} is not representable"))?, + ), + }, + Value::String(text) => ArgValue::Str(text.clone()), + other => return Err(format!("unsupported normalizeArgs input {other}")), + }) +} + +/// `normalizeArgsVectors`: build the host record (sentinel strings become +/// `Absent`, `bigintArgs` become `BigInt`, `specialArgs` become `Number`) and +/// require exactly the expected ordered pairs. +pub fn check_normalize_args_vector(vector: &Value) -> Result<(), String> { + let sentinel = vector.get("undefinedSentinel").and_then(Value::as_str); + let mut record: BTreeMap = BTreeMap::new(); + for (name, value) in vector["input"] + .as_object() + .ok_or("input is not an object")? + { + let arg = match (value.as_str(), sentinel) { + (Some(text), Some(sentinel)) if text == sentinel => ArgValue::Absent, + _ => arg_from_json(value)?, + }; + record.insert(name.clone(), arg); + } + if let Some(bigints) = vector.get("bigintArgs") { + for (name, text) in bigints.as_object().ok_or("bigintArgs is not an object")? { + let text = text + .as_str() + .ok_or_else(|| format!("bigintArgs {name} is not a string"))?; + record.insert(name.clone(), ArgValue::BigInt(text.to_owned())); + } + } + if let Some(specials) = vector.get("specialArgs") { + for (name, text) in specials.as_object().ok_or("specialArgs is not an object")? { + let text = text + .as_str() + .ok_or_else(|| format!("specialArgs {name} is not a string"))?; + let number: f64 = text + .parse() + .map_err(|error| format!("specialArgs {name} {text:?} is not a number: {error}"))?; + record.insert(name.clone(), ArgValue::Number(number)); + } + } + let expected: Vec<(String, String)> = serde_json::from_value(vector["expected"].clone()) + .map_err(|error| format!("expected pairs do not parse: {error}"))?; + let actual = + normalize_args(record).map_err(|error| format!("normalize_args failed: {error}"))?; + if actual != expected { + return Err(format!("normalized {actual:?} != expected {expected:?}")); + } + Ok(()) +} + +/// `rampVectors`: the cohort of the logical key and `layer` must equal `sample` +/// exactly, and the FNV-1a numerator must equal `hashNumerator` when present. +pub fn check_ramp_vector(vector: &Value) -> Result<(), String> { + let identity = identity_from(&vector["input"])?; + let keys = identity + .keys() + .map_err(|error| format!("keys rejected the ramp identity: {error}"))?; + let layer = vector["layer"].as_str().ok_or("layer missing")?; + let sample = vector["sample"].as_f64().ok_or("sample missing")?; + let actual = cohort(&keys.logical, layer); + if actual != sample { + return Err(format!("cohort {actual:.17} != expected {sample:.17}")); + } + if let Some(numerator) = vector.get("hashNumerator") { + let numerator = numerator + .as_u64() + .ok_or("hashNumerator is not an unsigned integer")?; + let hash = u64::from(cohort_hash(&keys.logical, layer)); + if hash != numerator { + return Err(format!("hash numerator {hash} != expected {numerator}")); + } + } + Ok(()) +} diff --git a/rust/tests/protocol_keys.rs b/rust/tests/protocol_keys.rs new file mode 100644 index 00000000..6b14551b --- /dev/null +++ b/rust/tests/protocol_keys.rs @@ -0,0 +1,51 @@ +//! Replays every key, invalid-key, argument-normalization and rollout vector +//! of the portable protocol corpus (fixed plus generated) through +//! `dialcache::identity`. +//! +//! Set `DIALCACHE_PROTOCOL_CORPUS=fixed|generated` to narrow the corpus. + +#[path = "formal/fixtures.rs"] +mod fixtures; +#[path = "formal/key_vectors.rs"] +mod key_vectors; + +use serde_json::Value; + +type Check = fn(&Value) -> Result<(), String>; + +const GROUPS: [(&str, Check); 4] = [ + ("keyVectors", key_vectors::check_key_vector), + ("invalidKeyVectors", key_vectors::check_invalid_key_vector), + ( + "normalizeArgsVectors", + key_vectors::check_normalize_args_vector, + ), + ("rampVectors", key_vectors::check_ramp_vector), +]; + +#[test] +fn key_protocol_vectors_replay() { + let groups = fixtures::protocol_groups("all"); + let mut failures = Vec::new(); + for (group, check) in GROUPS { + let rows = groups + .get(group) + .unwrap_or_else(|| panic!("protocol corpus has no {group} group")); + assert!(!rows.is_empty(), "{group}: no vectors selected"); + let mut passed = 0; + for row in rows { + let name = row["name"].as_str().unwrap_or(""); + match check(row) { + Ok(()) => passed += 1, + Err(reason) => failures.push(format!("{group} {name:?}: {reason}")), + } + } + println!("{group}: {passed}/{} passed", rows.len()); + } + assert!( + failures.is_empty(), + "{} vector(s) failed:\n{}", + failures.len(), + failures.join("\n") + ); +} From 7c1d4fd771a6b9b257bd537be571bc18ee8101da Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 00:21:30 -0700 Subject: [PATCH 03/31] feat(formal): integrate the Rust port into the shared conformance tooling Add the rust implementation to the shared coordinator protocol the way Go is integrated: an identity native binding, a strict JSONL report gate (check-rust-replay.mjs) and report adapter, default source bindings for the crate, and check-rust / formal-rust validation lanes with a cargo 1.98.1 prerequisite probe. The smoke target replays the committed histories through the Rust harness as well. Declare the rust implementation in profiles.json and refresh the two pins of its digest (generated-fixtures.lock.json inputs, go-parity.json inputs). Wire the pinned Rust toolchain into setup-validation, add the rust CI job and the rust-parity job of the full formal workflow, and pin the crate toolchain. --- .github/actions/setup-validation/action.yml | 14 ++ .github/workflows/ci.yaml | 19 +- .github/workflows/formal-full.yaml | 47 ++++- Makefile | 4 +- formal/check-rust-replay.mjs | 83 +++++++++ formal/conformance-adapters.mjs | 16 +- formal/conformance-bindings.mjs | 5 +- formal/conformance.mjs | 9 +- formal/generated-fixtures.lock.json | 2 +- formal/go-parity.json | 2 +- formal/profiles.json | 21 +++ formal/validation.mjs | 40 +++- rust/rust-toolchain.toml | 3 + test/formal-rust-replay.test.ts | 192 ++++++++++++++++++++ test/formal-validation.test.ts | 5 +- 15 files changed, 439 insertions(+), 23 deletions(-) create mode 100644 formal/check-rust-replay.mjs create mode 100644 rust/rust-toolchain.toml create mode 100644 test/formal-rust-replay.test.ts diff --git a/.github/actions/setup-validation/action.yml b/.github/actions/setup-validation/action.yml index ef185f4f..4cfda85e 100644 --- a/.github/actions/setup-validation/action.yml +++ b/.github/actions/setup-validation/action.yml @@ -10,6 +10,9 @@ inputs: java: description: Install Java 21 for bounded symbolic model checking. default: "false" + rust: + description: Install the pinned Rust toolchain (rust/rust-toolchain.toml) with clippy and rustfmt. + default: "false" runs: using: composite steps: @@ -35,3 +38,14 @@ runs: with: distribution: temurin java-version: "21" + # The toolchain pin matches rust/rust-toolchain.toml and the cargo 1.98.1 + # prerequisite probe in formal/validation.mjs. + - uses: dtolnay/rust-toolchain@02cb101ec7c40f2c49e1d9714d64511d8e1b74de # master 2026-09-12 + if: inputs.rust == 'true' + with: + toolchain: 1.98.1 + components: clippy, rustfmt + - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + if: inputs.rust == 'true' + with: + workspaces: rust diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 53a9fcfb..ca33fb73 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,17 +67,30 @@ jobs: retention-days: 14 if-no-files-found: warn - # The existing required `test` status now requires both native language jobs. + rust: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: ./.github/actions/setup-validation + with: + rust: "true" + - name: Check Rust formatting, clippy and native tests including the smoke conformance run + run: make check-rust + + # The existing required `test` status now requires every native language job. test: - needs: [typescript, go] + needs: [typescript, go, rust] if: always() runs-on: ubuntu-latest timeout-minutes: 5 steps: - - name: Require successful TypeScript and Go checks + - name: Require successful TypeScript, Go and Rust checks env: TYPESCRIPT_RESULT: ${{ needs.typescript.result }} GO_RESULT: ${{ needs.go.result }} + RUST_RESULT: ${{ needs.rust.result }} run: | test "$TYPESCRIPT_RESULT" = success test "$GO_RESULT" = success + test "$RUST_RESULT" = success diff --git a/.github/workflows/formal-full.yaml b/.github/workflows/formal-full.yaml index 66bc3e86..78d44dd2 100644 --- a/.github/workflows/formal-full.yaml +++ b/.github/workflows/formal-full.yaml @@ -247,6 +247,37 @@ jobs: if-no-files-found: warn retention-days: 14 + rust-parity: + needs: generate + runs-on: ubuntu-latest + # The Rust replay compiles the crate in release mode before replaying the + # same corpus Go completes in about 4.5 minutes; the budget mirrors go-parity. + timeout-minutes: 40 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: ./.github/actions/setup-validation + with: + rust: "true" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: formal-traces + path: .formal-traces + - name: Require Rust replay of the complete corpus + run: make formal-rust + - name: Preserve Rust parity completion evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: rust-parity-evidence + path: | + .formal-traces/rust-replay.jsonl + .formal-traces/rust-replay-summary.json + .formal-traces/rust-context.json + .formal-traces/rust-completion.json + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + # The Go lane bounded the whole workflow: 13 mutants each replaying the # generated cohort for about 105 s on a fast runner, strictly in sequence. # Run 34660598461 took 25 minutes; runs 34669546872 and 34670045249 took 47 @@ -316,7 +347,7 @@ jobs: # The mutation lanes count through their merge jobs: a merge succeeds only # when every shard's report is present, consistent and free of lost detections. formal-full: - needs: [check-models, generate, typescript-parity, symbolic, typescript-mutations-merge, go-parity, go-mutations-merge, exploration] + needs: [check-models, generate, typescript-parity, symbolic, typescript-mutations-merge, go-parity, go-mutations-merge, rust-parity, exploration] if: always() runs-on: ubuntu-latest timeout-minutes: 5 @@ -339,6 +370,13 @@ jobs: with: name: go-parity-evidence path: formal-summary/go + - name: Collect Rust completion evidence + if: always() + continue-on-error: true + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: rust-parity-evidence + path: formal-summary/rust - name: Collect model check report if: always() continue-on-error: true @@ -371,13 +409,16 @@ jobs: formal-summary/go/go-completion.json formal-summary/go/go-context.json formal-summary/go/go-replay-summary.json + formal-summary/rust/rust-completion.json + formal-summary/rust/rust-context.json + formal-summary/rust/rust-replay-summary.json formal-summary/model-check/model-properties/report.json formal-summary/symbolic/report.json formal-summary/exploration/*/report.json include-hidden-files: true if-no-files-found: warn retention-days: 90 - - name: Require the model check, the generated corpus, both port replays and both merged mutation measurements + - name: Require the model check, the generated corpus, every port replay and both merged mutation measurements env: CHECK_MODELS_RESULT: ${{ needs.check-models.result }} GENERATE_RESULT: ${{ needs.generate.result }} @@ -385,6 +426,7 @@ jobs: TYPESCRIPT_MUTATIONS_MERGE_RESULT: ${{ needs.typescript-mutations-merge.result }} GO_RESULT: ${{ needs.go-parity.result }} GO_MUTATIONS_MERGE_RESULT: ${{ needs.go-mutations-merge.result }} + RUST_RESULT: ${{ needs.rust-parity.result }} SYMBOLIC_RESULT: ${{ needs.symbolic.result }} EXPLORATION_RESULT: ${{ needs.exploration.result }} EXPLORATION_REQUIRED: ${{ github.event_name == 'schedule' || inputs.exploration == true }} @@ -395,6 +437,7 @@ jobs: test "$TYPESCRIPT_MUTATIONS_MERGE_RESULT" = success test "$GO_RESULT" = success test "$GO_MUTATIONS_MERGE_RESULT" = success + test "$RUST_RESULT" = success test "$SYMBOLIC_RESULT" = success if [ "$EXPLORATION_REQUIRED" = true ]; then test "$EXPLORATION_RESULT" = success diff --git a/Makefile b/Makefile index 47360da9..e63e0215 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ NODE ?= node -.PHONY: help check check-ts check-go docs audit smoke formal formal-check formal-generate formal-ts formal-go fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go package-floor ci explore model-check +.PHONY: help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go package-floor ci explore model-check -help check check-ts check-go docs audit smoke formal formal-check formal-generate formal-ts formal-go fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go package-floor ci explore model-check: +help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go package-floor ci explore model-check: $(NODE) formal/validation.mjs $@ diff --git a/formal/check-rust-replay.mjs b/formal/check-rust-replay.mjs new file mode 100644 index 00000000..64c00365 --- /dev/null +++ b/formal/check-rust-replay.mjs @@ -0,0 +1,83 @@ +import { createHash } from 'node:crypto'; +import { readFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { root } from './execution.mjs'; + +import { conformanceInventory } from './conformance.mjs'; +import { nativeBinding } from './conformance-bindings.mjs'; + +// The Rust harness reports one JSON object per line: a start record, one +// record per executed case named by its shared inventory id, and a finish +// record with totals. Case ids are the inventory ids themselves (the Rust +// binding is the identity), so this gate reads the report against the common +// inventory directly, with no second list of required behavior. +const inventoryRoots = new Set(['sampled', 'regression', 'scenario', 'protocol', 'witness']); +const epochMs = value => Number.isSafeInteger(value) && value > 0; + +/** Pure completed-report gate. Counts only exact passed case records, never totals the harness printed. */ +export function checkRustReplay(report, inventory = conformanceInventory()) { + if (typeof report !== 'string' || !report.trim()) throw new Error('Empty Rust replay report'); + if (!Array.isArray(inventory) || !inventory.length) throw new Error('Empty Rust replay inventory'); + const required = new Map(inventory.map(entry => [nativeBinding(entry, 'rust'), entry])); + if (required.size !== inventory.length) throw new Error('Duplicate Rust case binding'); + const cases = new Map(); + let started, finish; + for (const [index, line] of report.trim().split('\n').entries()) { + let event; + try { event = JSON.parse(line); } catch { throw new Error(`Invalid Rust JSON record at line ${index + 1}`); } + if (!event || typeof event !== 'object' || Array.isArray(event) || typeof event.kind !== 'string') throw new Error(`Invalid Rust replay record at line ${index + 1}`); + if (finish) throw new Error('Rust report continues after the finish record'); + if (event.kind === 'start') { + if (started || index !== 0) throw new Error('Duplicate or misplaced Rust start record'); + if (event.schemaVersion !== 1 || event.implementation !== 'rust' || !epochMs(event.startedAt)) throw new Error('Unsupported Rust start record'); + started = event; + continue; + } + if (!started) throw new Error('Rust replay record precedes the start record'); + if (event.kind === 'case') { + const { id } = event; + if (typeof id !== 'string' || !id) throw new Error(`Invalid Rust case id at line ${index + 1}`); + if (event.status !== 'passed' && event.status !== 'failed') throw new Error(`Invalid Rust case status: ${id}`); + if (!epochMs(event.startedAt) || !epochMs(event.finishedAt) || event.finishedAt < event.startedAt || event.startedAt < started.startedAt) throw new Error(`Invalid Rust case timing: ${id}`); + if (event.message !== undefined && typeof event.message !== 'string') throw new Error(`Invalid Rust case message: ${id}`); + if (event.status === 'failed') throw new Error(`Rust replay failed: ${id}${event.message ? ` (${event.message})` : ''}`); + if (cases.has(id)) throw new Error(`Duplicate Rust replay case: ${id}`); + if (!required.has(id)) { + const segment = id.split('/')[0]; + if (segment === 'smoke') throw new Error(`Smoke history in a full Rust replay: ${id}`); + if (inventoryRoots.has(segment)) throw new Error(`Unexpected Rust replay case (inventory drift): ${id}`); + throw new Error(`Unknown Rust replay case: ${id}`); + } + cases.set(id, event); + } else if (event.kind === 'finish') { + if (event.status !== 'passed' && event.status !== 'failed') throw new Error('Invalid Rust finish status'); + if (event.status !== 'passed') throw new Error(`Rust replay finished with status ${event.status}`); + if (!epochMs(event.finishedAt) || event.finishedAt < started.startedAt) throw new Error('Invalid Rust finish timing'); + if (event.cases !== cases.size || event.failed !== 0) throw new Error(`Rust finish totals disagree with the case records: ${event.cases} reported, ${cases.size} recorded, ${event.failed} failed`); + finish = event; + } else { + throw new Error(`Unsupported Rust replay record kind: ${event.kind}`); + } + } + if (!finish) throw new Error('Rust replay is incomplete: missing finish record'); + for (const id of required.keys()) if (!cases.has(id)) throw new Error(`Missing passed Rust replay case: ${id}`); + const profiles = [...new Set(inventory.filter(entry => entry.profile).map(entry => entry.profile))]; + const count = (category, profile) => inventory.filter(entry => entry.category === category && (profile === undefined || entry.profile === profile)).length; + const generated = Object.fromEntries(profiles.map(profile => [profile, count('sampled', profile)])); + const quintRegressions = Object.fromEntries(profiles.map(profile => [profile, count('regression', profile)])); + return { schemaVersion: 1, implementation: 'rust', status: 'pass', + generated, generatedTraces: Object.values(generated).reduce((sum, n) => sum + n, 0), + quintRegressions, quintRegressionTraces: Object.values(quintRegressions).reduce((sum, n) => sum + n, 0), + fixedScenarios: count('scenario'), protocolVectors: count('protocol'), + witnessProfiles: inventory.filter(entry => entry.category === 'witness').map(entry => entry.profile).sort(), + executedCases: cases.size }; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + if (process.argv.length > 3) throw new Error('Usage: node formal/check-rust-replay.mjs [rust-replay.jsonl]'); + const path = process.argv[2] ? resolve(process.argv[2]) : root + '.formal-traces/rust-replay.jsonl'; + const report = readFileSync(path, 'utf8'); + const result = checkRustReplay(report, conformanceInventory()); + console.log(JSON.stringify({ ...result, reportSha256: createHash('sha256').update(report).digest('hex') }, null, 2)); +} diff --git a/formal/conformance-adapters.mjs b/formal/conformance-adapters.mjs index f6c86376..03f8d9f7 100644 --- a/formal/conformance-adapters.mjs +++ b/formal/conformance-adapters.mjs @@ -2,6 +2,7 @@ import { readFileSync } from 'node:fs'; import { basename, resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; import { checkGoReplay, loadGoReplayInventory } from './check-go-replay.mjs'; +import { checkRustReplay } from './check-rust-replay.mjs'; import { root } from './execution.mjs'; import { readJSON, digest, fingerprint, validateContext, checkCompletion } from './conformance.mjs'; @@ -51,11 +52,22 @@ export function parseGoReport(text, inventory) { }) }; } +// The Rust harness names cases by inventory id, so the strict report gate is +// also the binding check; only the execution window is read here. +export function parseRustReport(text, inventory) { + checkRustReplay(text, inventory); + const events = text.trim().split('\n').map(line => JSON.parse(line)); + const startedAt = events[0].startedAt, finishedAt = events.at(-1).finishedAt; + if (!Number.isFinite(startedAt) || !Number.isFinite(finishedAt)) throw new Error('Rust report is missing execution timestamps'); + return { startedAt, finishedAt, results: inventory.map(entry => ({ id: entry.id, status: 'passed' })) }; +} + export function adaptReport(language, text, context) { validateContext(context); if (context.language !== language) throw new Error('Wrong port context'); const parsed = language === 'typescript' ? parseTypeScriptReport(text, context.inventory) : language === 'go' - ? parseGoReport(text, context.inventory) : (() => { throw new Error('Unsupported native report adapter'); })(); + ? parseGoReport(text, context.inventory) : language === 'rust' ? parseRustReport(text, context.inventory) + : (() => { throw new Error('Unsupported native report adapter'); })(); const report = { schemaVersion: 1, language, runId: context.runId, contextSha256: fingerprint(context), ...parsed, status: 'passed', nativeReportSha256: digest(text) }; checkCompletion(report, context); @@ -63,6 +75,6 @@ export function adaptReport(language, text, context) { } if (process.argv[1] === fileURLToPath(import.meta.url)) { const [language, nativePath, contextPath, ...extra] = process.argv.slice(2); - if (extra.length || !language || !nativePath || !contextPath) throw new Error('Usage: node formal/conformance-adapters.mjs '); + if (extra.length || !language || !nativePath || !contextPath) throw new Error('Usage: node formal/conformance-adapters.mjs '); console.log(JSON.stringify(adaptReport(language, readFileSync(resolve(root, nativePath), 'utf8'), readJSON(contextPath)), null, 2)); } diff --git a/formal/conformance-bindings.mjs b/formal/conformance-bindings.mjs index 3af69129..9efe96f8 100644 --- a/formal/conformance-bindings.mjs +++ b/formal/conformance-bindings.mjs @@ -37,7 +37,10 @@ export function nativeBinding(entry, language, workspace = root) { if (entry.category === 'protocol') return `${protocolGoRoots[entry.group] ?? 'TestProtocolRemainingVectors'}/${goName(entry.name)}`; return `TestGeneratedWitnessEvidence/${entry.profile}`; } - if (language !== 'typescript') throw new Error('Native report adapter is only supplied for TypeScript and Go'); + // The Rust harness names every case by its shared inventory id, so the + // binding is the identity: the report is read against the inventory directly. + if (language === 'rust') return entry.id; + if (language !== 'typescript') throw new Error('Native report adapter is only supplied for TypeScript, Go and Rust'); if (entry.category === 'sampled' || entry.category === 'regression') return [profileFile(entry.profile), `${profileSuite(entry.profile)} replays ${resolve(workspace, entry.path)}`]; if (entry.category === 'scenario') return ['formal-behavior.test.ts', `portable behavioral scenarios ${entry.feature}: ${entry.name}`]; diff --git a/formal/conformance.mjs b/formal/conformance.mjs index 49a11af8..07d5cef1 100644 --- a/formal/conformance.mjs +++ b/formal/conformance.mjs @@ -51,6 +51,13 @@ export function defaultSources(language) { ...filesBelow('go').filter(path => /\.(go|ts)$/.test(path) || /\/go\.(mod|sum)$/.test(path)), ...readExecution().models.filter(model => model.profile && model.profile !== 'core') .map(model => `.formal-traces/go-parity-witnesses/${model.profile}.json`)]; + // Rust binds the same shared fixtures, its crate sources, manifests and + // lockfile, and the same witness evidence the Go replay consumes. The build + // directory is a cache, never an input. + if (language === 'rust') return [...shared, + ...filesBelow('rust').filter(path => !path.startsWith('rust/target/') && /\.(rs|toml|lock)$/.test(path)), + ...readExecution().models.filter(model => model.profile && model.profile !== 'core') + .map(model => `.formal-traces/go-parity-witnesses/${model.profile}.json`)]; fail('New languages must supply an explicit JSON list of implementation and harness source paths'); } function hashes(paths) { @@ -97,7 +104,7 @@ export function validateContext(context, { current = true } = {}) { if (!isDeepStrictEqual(context.implementation, hashes(Object.keys(context.implementation)))) fail('Implementation inputs changed during run'); // Default bindings must include new files too; custom port inventories are // an explicit, reviewable declaration of the complete execution inputs. - if (['typescript', 'go'].includes(context.language) && !isDeepStrictEqual(Object.keys(context.implementation).sort(), defaultSources(context.language).sort())) fail('Implementation source inventory changed during run'); + if (['typescript', 'go', 'rust'].includes(context.language) && !isDeepStrictEqual(Object.keys(context.implementation).sort(), defaultSources(context.language).sort())) fail('Implementation source inventory changed during run'); if (!isDeepStrictEqual(context.corpus, corpusInputs(context.inventory))) fail('Shared corpus changed during run'); } return context; diff --git a/formal/generated-fixtures.lock.json b/formal/generated-fixtures.lock.json index 4b8e7dd2..8c3d08e6 100644 --- a/formal/generated-fixtures.lock.json +++ b/formal/generated-fixtures.lock.json @@ -41,7 +41,7 @@ "formal/kernel/request-memo.qnt": "e7897bfe9949a4d31b8138b35c32a7df0939e81e1f9c13739340a794a3d5073f", "formal/kernel/runtime-policy.qnt": "41ebb180f8b8b561d0db43b237abbee04e69613521436f6c624241cfece536ae", "formal/kernel/serving.qnt": "f3e669099d0cff1b6d243c8822ea07e66b90ec283475646c9a5bac094c173f00", - "formal/profiles.json": "b8fa4f399a163bc8793bf76b6ba3503a415fcf3c8509cf98d964e4f067e6660f", + "formal/profiles.json": "ce15fe75cbd465711fbad84fca9a10fe8a890469bd8fb665522bb6837ab71dec", "formal/wire-text.qnt": "c78f8c674f4e1470b0d2b605b209d82e3aab0e35fb4e0efae813247bf093eb27" }, "artifacts": { diff --git a/formal/go-parity.json b/formal/go-parity.json index 5fdf7b37..ea89824e 100644 --- a/formal/go-parity.json +++ b/formal/go-parity.json @@ -14,7 +14,7 @@ "quintCaseAuditSha256": "dcb576948a68a066e43259d4bee2b3cdadbe82e657ae9af99e4b21ed7b95dbb3", "featureCoverageSha256": "56eedfa865682db6ad8e73b8673c7dac1c30d301e9bce3390246d6994aba9dcb", "coverageWitnessesSha256": "8fc8ab123150fdec059f5b75f84172d39af80d08d7a4211fa88c63c5a3f2874b", - "profilesSha256": "b8fa4f399a163bc8793bf76b6ba3503a415fcf3c8509cf98d964e4f067e6660f" + "profilesSha256": "ce15fe75cbd465711fbad84fca9a10fe8a890469bd8fb665522bb6837ab71dec" }, "inventory": { "behavioralCases": 240, diff --git a/formal/profiles.json b/formal/profiles.json index 17fa6e2e..7e44ea97 100644 --- a/formal/profiles.json +++ b/formal/profiles.json @@ -303,6 +303,27 @@ ], "definition": "go/README.md", "limits": "Same generated histories and portable scenarios as TypeScript, all protocol vectors, plus native Redis/Valkey/Cluster and cross-language integration. Claims require current corpus and witness fingerprints; finite evidence does not prove every possible schedule. Native API and value adaptations are documented in the Go README and parity ledger." + }, + { + "id": "rust", + "profiles": [ + "core", + "effects", + "scope", + "recovery", + "policy", + "shadow", + "admission", + "layers", + "independent", + "recovery-read", + "local-failure", + "runtime-boundaries", + "shadow-layers", + "local-clock", + "source-budgets" + ], + "limits": "Same generated histories and portable scenarios as TypeScript and Go and all protocol vectors, replayed by the Rust conformance harness against the shared witness evidence. Claims require current corpus and witness fingerprints; finite evidence does not prove every possible schedule. Native Redis integration is not yet part of the claim." } ], "replaySources": [ diff --git a/formal/validation.mjs b/formal/validation.mjs index dbef55c9..1d9e4d8d 100644 --- a/formal/validation.mjs +++ b/formal/validation.mjs @@ -8,24 +8,26 @@ const root = fileURLToPath(new URL('../', import.meta.url)); const replayTests = ['test/formal-conformance.test.ts', 'test/formal-effects.test.ts', 'test/formal-features.test.ts', 'test/formal-local-clock.test.ts', 'test/formal-behavior.test.ts', 'test/formal-protocol-vectors.test.ts']; const aggregateTargets = { - check: ['check-ts', 'check-go', 'docs', 'audit'], - formal: ['formal-check', 'formal-generate', 'formal-ts', 'formal-go'], + check: ['check-ts', 'check-go', 'check-rust', 'docs', 'audit'], + formal: ['formal-check', 'formal-generate', 'formal-ts', 'formal-go', 'formal-rust'], mutations: ['mutations-ts', 'mutations-go'], integration: ['integration-ts', 'integration-go'], ci: ['check', 'package-floor', 'formal', 'model-check', 'integration', 'mutations'], }; export const targetDescriptions = { - check: 'TypeScript and Go checks, docs build and reviewed inventories; no Quint generation or Docker', + check: 'TypeScript, Go and Rust checks, docs build and reviewed inventories; no Quint generation or Docker', 'check-ts': 'Typecheck, unit coverage, build and packed-package checks on Node 24', 'check-go': 'Go vet, formatting check and default tests with race detection', + 'check-rust': 'Rust formatting check, clippy with warnings denied and default tests including the smoke conformance run', docs: 'Build the documentation site', audit: 'Check source, behavior, feature, Go and generated-fixture freshness inventories', - smoke: 'Replay committed Quint-derived fixtures in TypeScript and Go; no full completion claim', - formal: 'Check every scheduled Quint model, generate the complete corpus and shared witness evidence, then complete TypeScript and Go replay', + smoke: 'Replay committed Quint-derived fixtures in TypeScript, Go and Rust; no full completion claim', + formal: 'Check every scheduled Quint model, generate the complete corpus and shared witness evidence, then complete TypeScript, Go and Rust replay', 'formal-check': 'Typecheck and run every scheduled Quint model, its public regressions and the model mutation challenges', 'formal-generate': 'Generate/recompute artifacts and evaluate shared witness evidence over the complete corpus', 'formal-ts': 'Complete prepared TypeScript replay of the generated corpus', 'formal-go': 'Complete prepared Go replay of the generated corpus with race detection', + 'formal-rust': 'Complete prepared Rust replay of the generated corpus in release mode', 'fixtures-check': 'Recompute every committed model-derived artifact with pinned Quint', 'kernel-fixtures': 'Typecheck the kernel library fixtures (test/fixtures/kernel) and run every run they declare', differential: 'Check the composition lint baseline, then replay every composed profile against its reference corpus (merge base with DIFFERENTIAL_REFERENCE, default origin/main) in both directions', @@ -93,6 +95,8 @@ export function validationPlan(target, { directory = root, environment = process const node = (label, ...args) => ({ label, command: runnerNode, args }); const pnpm = (label, ...args) => ({ label, command: 'corepack', args: ['pnpm', ...args] }); const go = (label, ...args) => ({ label, command: 'go', args: ['-C', 'go', ...args] }); + // Every step spawns from the repository root; the manifest path selects the crate. + const cargo = (label, subcommand, ...args) => ({ label, command: 'cargo', args: [subcommand, '--manifest-path', 'rust/Cargo.toml', ...args] }); const reportPath = (language, suffix) => `.formal-traces/${language}-${suffix}.json`; const completion = language => ({ ...node(`Validate current ${language} completion`, 'formal/conformance.mjs', 'check', reportPath(language, 'completion'), reportPath(language, 'context')), failureHint: 'A current complete replay is required. Run make formal first; missing or stale reports cannot be reused.' }); @@ -122,18 +126,28 @@ export function validationPlan(target, { directory = root, environment = process const nativeGo = full => ({ ...go(full ? 'Replay complete Go corpus with race detection' : 'Run Go default tests with race detection', 'test', '-race', '-count=1', ...(full ? ['-json', '-timeout=35m'] : []), './...'), ...(full ? { env: { ...replayEnv, DIALCACHE_WITNESS_EVIDENCE_DIR: witnessDirectory }, stdoutFile: '.formal-traces/go-replay.jsonl' } : {}) }); + // The Rust conformance harness is one cargo test target. Without directory + // selectors it replays the committed smoke histories; with them it replays + // the complete corpus and writes its own JSONL report to DIALCACHE_RUST_REPORT + // (cargo's stdout carries build output, so it is not the report channel). + const nativeRust = full => ({ ...cargo(full ? 'Replay complete Rust corpus' : 'Replay committed Rust fixtures', + 'test', ...(full ? ['--release'] : []), '--all-features', '--test', 'conformance'), + ...(full ? { env: { ...replayEnv, DIALCACHE_WITNESS_EVIDENCE_DIR: witnessDirectory, DIALCACHE_RUST_REPORT: resolve(directory, '.formal-traces/rust-replay.jsonl') } } : {}) }); const node22 = floorExecutable(environment, runnerNode, nodeVersion) ?? ''; const shard = mutationShardArguments(target, environment); const plans = { 'check-ts': [pnpm('Typecheck TypeScript', 'typecheck'), pnpm('Run TypeScript unit tests with coverage', 'test'), pnpm('Build package', 'build'), pnpm('Check packed package on Node 24', 'test:package')], 'check-go': [go('Go vet', 'vet', './...'), { label: 'Check Go formatting', command: 'gofmt', args: ['-l', 'go'], requireEmptyStdout: true }, nativeGo(false)], + 'check-rust': [cargo('Check Rust formatting', 'fmt', '--check'), + cargo('Lint Rust with clippy', 'clippy', '--all-targets', '--all-features', '--', '-D', 'warnings'), + cargo('Run Rust default tests', 'test', '--all-features')], docs: [pnpm('Build documentation', 'docs:build')], audit: ['execution', 'check-source-audit', 'check-semantic-coverage', 'check-feature-coverage', 'check-go-parity'] .map(name => node(`Check ${name}`, `formal/${name}.mjs`)) .concat(node('Verify committed fixture fingerprints', 'formal/generated-fixtures.mjs', '--verify'), node('Check conditional fixture regeneration scope', '--test', '.github/scripts/fixture-scope.test.mjs')), - smoke: [tsReplay(false), nativeGo(false)], + smoke: [tsReplay(false), nativeGo(false), nativeRust(false)], 'fixtures-check': [node('Recompute all committed Quint artifacts', 'formal/generate-artifacts.mjs', '--check')], 'kernel-fixtures': [kernelFixtures], differential: [lintBaseline, kernelFixtures, node('Replay composed profiles against their reference corpus', 'formal/differential.mjs', '--composed', `--reference=${environment.DIFFERENTIAL_REFERENCE ?? 'origin/main'}`)], @@ -146,7 +160,7 @@ export function validationPlan(target, { directory = root, environment = process // Generation is the single shared producer: the corpus, wire artifacts and // witness evidence depend only on the models. Both ports' replays and both // mutation measurements read that output and can run in parallel off it. - 'formal-generate': [invalidate('ts', 'go'), + 'formal-generate': [invalidate('ts', 'go', 'rust'), node('Generate complete corpus and recompute wire artifacts', 'formal/run-models.mjs', 'generate'), node('Recompute committed Quint smoke and witness fixtures', 'formal/generated-fixtures.mjs', '--check'), witnesses], 'formal-ts': [invalidate('ts'), node('Prepare TypeScript execution context', 'formal/conformance.mjs', 'prepare', 'typescript', reportPath('ts', 'context')), @@ -156,6 +170,12 @@ export function validationPlan(target, { directory = root, environment = process node('Prepare Go execution context', 'formal/conformance.mjs', 'prepare', 'go', reportPath('go', 'context')), nativeGo(true), { ...node('Check complete Go native report', 'formal/check-go-replay.mjs'), stdoutFile: '.formal-traces/go-replay-summary.json' }, { ...node('Adapt Go native assertion report', 'formal/conformance-adapters.mjs', 'go', '.formal-traces/go-replay.jsonl', reportPath('go', 'context')), stdoutFile: reportPath('go', 'completion') }, completion('go')], + // The Go parity ledger is Go-only; Rust has no ledger step. The harness + // names cases by inventory id, so the report gate checks the binding too. + 'formal-rust': [invalidate('rust'), + node('Prepare Rust execution context', 'formal/conformance.mjs', 'prepare', 'rust', reportPath('rust', 'context')), nativeRust(true), + { ...node('Check complete Rust native report', 'formal/check-rust-replay.mjs'), stdoutFile: '.formal-traces/rust-replay-summary.json' }, + { ...node('Adapt Rust native assertion report', 'formal/conformance-adapters.mjs', 'rust', '.formal-traces/rust-replay.jsonl', reportPath('rust', 'context')), stdoutFile: reportPath('rust', 'completion') }, completion('rust')], 'mutations-ts': [node('Measure TypeScript semantic mutations', 'formal/measure-semantics.mjs', ...shard)], 'mutations-go': [node('Measure Go semantic mutations', 'formal/measure-go-semantics.mjs', ...shard)], // The merge needs neither Quint nor Go: it reads shard reports, checks @@ -188,6 +208,10 @@ export function checkPrerequisites(target, { directory = root, environment = pro const version = probe('go', ['version'], { directory, environment }); if (!/^go version go1\.27\.1\s/.test(version)) throw new Error(`Validation requires Go 1.27.1; found ${version}. Put the pinned Go toolchain on PATH.`); } + if (targets.some(name => ['check-rust', 'smoke', 'formal-rust'].includes(name))) { + const version = probe('cargo', ['--version'], { directory, environment }); + if (!/^cargo 1\.98\.1(?:\s|$)/.test(version)) throw new Error(`Validation requires cargo 1.98.1; found ${version}. Put the pinned Rust toolchain (rust/rust-toolchain.toml) on PATH.`); + } if (targets.some(name => ['formal-check', 'formal-generate', 'fixtures-check', 'explore', 'model-check', 'differential'].includes(name))) { const requiredQuint = JSON.parse(readFileSync(resolve(directory, 'formal/generated-fixtures.lock.json'), 'utf8')).quintVersion; const version = probe('quint', ['--version'], { directory, environment }); @@ -349,7 +373,7 @@ if (process.argv[1] === fileURLToPath(import.meta.url)) { if (target === 'help') { const width = Math.max(...Object.keys(targetDescriptions).map(name => name.length)); console.log(Object.entries(targetDescriptions).map(([name, description]) => `make ${name.padEnd(width)} ${description}`).join('\n')); - console.log('\nPrerequisites: frozen pnpm install; Node 24, pinned pnpm; Go 1.27.1 / Docker where required; Quint 0.32.0 for formal-check, formal-generate, fixtures-check, explore and model-check; Java 21 and tar for model-check and ci.'); + console.log('\nPrerequisites: frozen pnpm install; Node 24, pinned pnpm; Go 1.27.1 / cargo 1.98.1 / Docker where required; Quint 0.32.0 for formal-check, formal-generate, fixtures-check, explore and model-check; Java 21 and tar for model-check and ci.'); console.log('formal-check is the Quint evidence lane (models, regressions, challenges); the port and mutation lanes read only the formal-generate output and do not wait for it.'); console.log('Sharded mutation runs: MUTATION_SHARD=1/3 make mutations-ts (then 2/3, 3/3, on any machines with the same corpus), then make mutations-merge-ts; the merged report is the only complete evidence.'); console.log('Full local CI: make ci NODE22_BIN=/absolute/path/to/node22/bin/node (exact 22.15.0).'); diff --git a/rust/rust-toolchain.toml b/rust/rust-toolchain.toml new file mode 100644 index 00000000..ef662bfe --- /dev/null +++ b/rust/rust-toolchain.toml @@ -0,0 +1,3 @@ +[toolchain] +channel = "1.98.1" +components = ["clippy", "rustfmt"] diff --git a/test/formal-rust-replay.test.ts b/test/formal-rust-replay.test.ts new file mode 100644 index 00000000..033cfacb --- /dev/null +++ b/test/formal-rust-replay.test.ts @@ -0,0 +1,192 @@ +import { describe, expect, it } from "vitest"; + +type Entry = { id: string; category: string; profile?: string; path?: string; name?: string; feature?: string; group?: string }; +type Summary = { + schemaVersion: number; implementation: string; status: string; generated: Record; generatedTraces: number; + quintRegressions: Record; quintRegressionTraces: number; fixedScenarios: number; protocolVectors: number; + witnessProfiles: string[]; executedCases: number; +}; +type Step = { label: string; command?: string; args?: string[]; env?: NodeJS.ProcessEnv; stdoutFile?: string; remove?: string[] }; +type Record_ = Record; + +const { checkRustReplay } = await import(new URL("../formal/check-rust-replay.mjs", import.meta.url).href) as { + checkRustReplay(report: string, inventory: Entry[]): Summary; +}; +const { parseRustReport, adaptReport, nativeBinding } = await import(new URL("../formal/conformance-adapters.mjs", import.meta.url).href) as { + parseRustReport(text: string, inventory: Entry[]): { startedAt: number; finishedAt: number; results: Array<{ id: string; status: string }> }; + adaptReport(language: string, text: string, context: unknown): unknown; + nativeBinding(entry: Entry, language: string): unknown; +}; +const { conformanceInventory, defaultSources } = await import(new URL("../formal/conformance.mjs", import.meta.url).href) as { + conformanceInventory(): Entry[]; + defaultSources(language: string): string[]; +}; +const { validationPlan, checkPrerequisites } = await import(new URL("../formal/validation.mjs", import.meta.url).href) as { + validationPlan(target: string, options?: { directory?: string; environment?: NodeJS.ProcessEnv }): Step[]; + checkPrerequisites(target: string, options?: { directory?: string; environment?: NodeJS.ProcessEnv; nodeVersion?: string }): void; +}; + +const inventory = conformanceInventory(); +const startedAt = 1_800_000_000_000; +const start = (): Record_ => ({ schemaVersion: 1, kind: "start", implementation: "rust", startedAt }); +const caseRecord = (id: string, status = "passed", extra: Record_ = {}): Record_ => ({ kind: "case", id, status, startedAt: startedAt + 1, finishedAt: startedAt + 2, ...extra }); +const finish = (cases: number, failed = 0, status = failed ? "failed" : "passed"): Record_ => ({ kind: "finish", status, finishedAt: startedAt + 3, cases, failed }); +const encode = (records: unknown[]): string => records.map(record => JSON.stringify(record)).join("\n") + "\n"; +const completed = (): Record_[] => [start(), ...inventory.map(entry => caseRecord(entry.id)), finish(inventory.length)]; +const withCases = (mutate: (records: Record_[]) => void): string => { + const records = completed(); + mutate(records); + return encode(records); +}; + +describe("Rust replay report gate", () => { + it("binds every inventory entry to its own id", () => { + for (const entry of inventory) expect(nativeBinding(entry, "rust")).toBe(entry.id); + expect(nativeBinding(inventory[0]!, "go")).not.toBe(inventory[0]!.id); + expect(() => nativeBinding(inventory[0]!, "zig")).toThrow(/TypeScript, Go and Rust/); + }); + + it("accepts a complete report and summarizes it by category and profile", () => { + const summary = checkRustReplay(encode(completed()), inventory); + const count = (category: string) => inventory.filter(entry => entry.category === category).length; + expect(summary).toMatchObject({ schemaVersion: 1, implementation: "rust", status: "pass", executedCases: inventory.length, + generatedTraces: count("sampled"), quintRegressionTraces: count("regression"), fixedScenarios: count("scenario"), protocolVectors: count("protocol") }); + expect(summary.witnessProfiles).toEqual(inventory.filter(entry => entry.category === "witness").map(entry => entry.profile).sort()); + expect(Object.values(summary.generated).reduce((sum, n) => sum + n, 0)).toBe(summary.generatedTraces); + expect(summary.generated.core).toBeGreaterThan(0); + expect(summary.quintRegressions.core).toBe(inventory.filter(entry => entry.category === "regression" && entry.profile === "core").length); + for (const profile of Object.keys(summary.generated)) expect(summary.quintRegressions, profile).toHaveProperty(profile); + }); + + it("rejects a missing, duplicate or failed case", () => { + const missing = inventory.at(-1)!.id; + expect(() => checkRustReplay(withCases(records => { records.splice(records.length - 2, 1); (records.at(-1) as { cases: number }).cases--; }), inventory)) + .toThrow(`Missing passed Rust replay case: ${missing}`); + expect(() => checkRustReplay(withCases(records => { records.splice(1, 0, caseRecord(inventory[0]!.id)); (records.at(-1) as { cases: number }).cases++; }), inventory)) + .toThrow(`Duplicate Rust replay case: ${inventory[0]!.id}`); + expect(() => checkRustReplay(withCases(records => { records[1] = caseRecord(inventory[0]!.id, "failed", { message: "observation mismatch at step 3" }); }), inventory)) + .toThrow(`Rust replay failed: ${inventory[0]!.id} (observation mismatch at step 3)`); + expect(() => checkRustReplay(withCases(records => { records[1] = caseRecord(inventory[0]!.id, "skipped"); }), inventory)).toThrow(/Invalid Rust case status/); + }); + + it("rejects inventory drift, smoke histories in a full replay and unknown ids", () => { + const add = (id: string) => withCases(records => { records.splice(1, 0, caseRecord(id)); (records.at(-1) as { cases: number }).cases++; }); + expect(() => checkRustReplay(add("sampled/core/999999"), inventory)).toThrow("Unexpected Rust replay case (inventory drift): sampled/core/999999"); + expect(() => checkRustReplay(add("witness/nonexistent"), inventory)).toThrow(/inventory drift/); + expect(() => checkRustReplay(add("smoke/core/conformance-smoke.itf.json"), inventory)).toThrow("Smoke history in a full Rust replay: smoke/core/conformance-smoke.itf.json"); + expect(() => checkRustReplay(add("bench/unrelated"), inventory)).toThrow("Unknown Rust replay case: bench/unrelated"); + }); + + it("rejects an incomplete report, a non-passed finish, inconsistent totals and malformed lines", () => { + expect(() => checkRustReplay(withCases(records => { records.pop(); }), inventory)).toThrow("Rust replay is incomplete: missing finish record"); + expect(() => checkRustReplay(withCases(records => { records[records.length - 1] = finish(inventory.length, 0, "failed"); }), inventory)).toThrow("Rust replay finished with status failed"); + expect(() => checkRustReplay(withCases(records => { records[records.length - 1] = finish(inventory.length, 1, "passed"); }), inventory)).toThrow(/finish totals disagree/); + expect(() => checkRustReplay(withCases(records => { records[records.length - 1] = finish(inventory.length - 1); }), inventory)).toThrow(/finish totals disagree/); + expect(() => checkRustReplay(withCases(records => { records.push(caseRecord(inventory[0]!.id)); }), inventory)).toThrow(/continues after the finish record/); + expect(() => checkRustReplay(withCases(records => { records.shift(); }), inventory)).toThrow(/precedes the start record/); + expect(() => checkRustReplay(withCases(records => { records.splice(1, 0, start()); }), inventory)).toThrow(/Duplicate or misplaced Rust start record/); + expect(() => checkRustReplay(withCases(records => { records[0] = { ...start(), implementation: "go" }; }), inventory)).toThrow(/Unsupported Rust start record/); + expect(() => checkRustReplay(encode(completed()).replace("\n", "\n{not json\n"), inventory)).toThrow("Invalid Rust JSON record at line 2"); + expect(() => checkRustReplay(withCases(records => { records[1] = { kind: "note", id: "x" }; }), inventory)).toThrow(/Unsupported Rust replay record kind: note/); + expect(() => checkRustReplay("", inventory)).toThrow(/Empty Rust replay report/); + }); + + it("adapts the execution window and results only from a report that passed the gate", () => { + const parsed = parseRustReport(encode(completed()), inventory); + expect(parsed).toMatchObject({ startedAt, finishedAt: startedAt + 3 }); + expect(parsed.results).toEqual(inventory.map(entry => ({ id: entry.id, status: "passed" }))); + expect(() => parseRustReport(withCases(records => { records.pop(); }), inventory)).toThrow(/missing finish/); + expect(() => adaptReport("rust", encode(completed()), { schemaVersion: 1, language: "go" })).toThrow(/context/); + }); + + it("binds the crate sources, manifests and lockfile with the shared fixtures and witness evidence, never the build directory", () => { + const sources = defaultSources("rust"); + expect(new Set(sources).size).toBe(sources.length); + expect(sources).toContain("rust/Cargo.toml"); + expect(sources).toContain("rust/Cargo.lock"); + expect(sources).toContain("rust/rust-toolchain.toml"); + expect(sources).toContain("rust/src/lib.rs"); + expect(sources).toContain("rust/tests/conformance.rs"); + expect(sources.some(path => path.startsWith("rust/target/"))).toBe(false); + expect(sources.filter(path => path.startsWith("rust/")).every(path => /\.(rs|toml|lock)$/.test(path))).toBe(true); + const go = defaultSources("go"); + for (const path of go.filter(path => path.startsWith(".formal-traces/go-parity-witnesses/") || path.startsWith("src/") || path.startsWith("test/"))) { + expect(sources, path).toContain(path); + } + }); +}); + +describe("Rust validation lanes", () => { + const directory = "/checkout"; + it("checks formatting, clippy and default tests from the repository root through the crate manifest", () => { + expect(validationPlan("check-rust", { directory }).map(step => [step.command, ...step.args!])).toEqual([ + ["cargo", "fmt", "--manifest-path", "rust/Cargo.toml", "--check"], + ["cargo", "clippy", "--manifest-path", "rust/Cargo.toml", "--all-targets", "--all-features", "--", "-D", "warnings"], + ["cargo", "test", "--manifest-path", "rust/Cargo.toml", "--all-features"], + ]); + for (const step of validationPlan("check-rust", { directory })) expect(step.env, step.label).toBeUndefined(); + expect(validationPlan("check", { directory })).toEqual(["check-ts", "check-go", "check-rust", "docs", "audit"].flatMap(target => validationPlan(target, { directory }))); + }); + + it("replays the complete corpus against the shared evidence and adapts the harness report into a completion", () => { + const plan = validationPlan("formal-rust", { directory }); + expect(plan[0]).toEqual({ label: "Invalidate prior rust completion", remove: [".formal-traces/rust-completion.json"] }); + expect(plan[1]!.args).toEqual(["formal/conformance.mjs", "prepare", "rust", ".formal-traces/rust-context.json"]); + const replay = plan[2]!; + expect([replay.command, ...replay.args!]).toEqual(["cargo", "test", "--manifest-path", "rust/Cargo.toml", "--release", "--all-features", "--test", "conformance"]); + expect(replay.env).toEqual({ + DIALCACHE_MBT_TRACE_DIR: "/checkout/.formal-traces/conformance", + DIALCACHE_EFFECTS_TRACE_DIR: "/checkout/.formal-traces/effects", + DIALCACHE_FEATURE_TRACE_DIR: "/checkout/.formal-traces/features", + DIALCACHE_WITNESS_EVIDENCE_DIR: "/checkout/.formal-traces/go-parity-witnesses", + DIALCACHE_RUST_REPORT: "/checkout/.formal-traces/rust-replay.jsonl", + }); + expect(replay.stdoutFile).toBeUndefined(); + expect(plan[3]).toMatchObject({ args: ["formal/check-rust-replay.mjs"], stdoutFile: ".formal-traces/rust-replay-summary.json" }); + expect(plan[4]).toMatchObject({ args: ["formal/conformance-adapters.mjs", "rust", ".formal-traces/rust-replay.jsonl", ".formal-traces/rust-context.json"], stdoutFile: ".formal-traces/rust-completion.json" }); + expect(plan.at(-1)!.args).toEqual(["formal/conformance.mjs", "check", ".formal-traces/rust-completion.json", ".formal-traces/rust-context.json"]); + expect(plan).toHaveLength(6); + // Go's parity ledger is Go-only; Rust neither checks it nor touches the other ports' reports. + expect(plan.some(step => step.args?.[0] === "formal/check-go-parity.mjs")).toBe(false); + expect(plan.some(step => step.args?.some(argument => /\.formal-traces\/(ts|go)-/.test(argument)))).toBe(false); + const go = validationPlan("formal-go", { directory }).find(step => step.command === "go" && step.env)!; + for (const key of Object.keys(go.env!)) expect(replay.env![key], key).toBe(go.env![key]); + expect(validationPlan("formal", { directory }).slice(-plan.length)).toEqual(plan); + }); + + it("adds the smoke conformance run in default mode with no corpus selectors", () => { + const smoke = validationPlan("smoke", { directory }); + expect(smoke.at(-1)).toEqual({ label: "Replay committed Rust fixtures", command: "cargo", args: ["test", "--manifest-path", "rust/Cargo.toml", "--all-features", "--test", "conformance"] }); + expect(smoke.filter(step => step.command === "cargo")).toHaveLength(1); + }); + + it("probes the pinned cargo exactly for the Rust lanes", async () => { + const { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } = await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { delimiter, join } = await import("node:path"); + const temporary = mkdtempSync(join(tmpdir(), "dialcache-rust-prereq-")); + try { + for (const path of ["bin", "formal", "node_modules/typescript"]) mkdirSync(join(temporary, path), { recursive: true }); + writeFileSync(join(temporary, "package.json"), '{"packageManager":"pnpm@10.33.0"}'); + writeFileSync(join(temporary, "node_modules/typescript/package.json"), "{}"); + writeFileSync(join(temporary, "formal/generated-fixtures.lock.json"), '{"quintVersion":"0.32.0"}'); + const tool = (name: string, body: string) => { const path = join(temporary, "bin", name); writeFileSync(path, `#!${process.execPath}\n${body}\n`); chmodSync(path, 0o755); }; + tool("corepack", 'console.log("10.33.0")'); + tool("go", 'console.log("go version go1.27.1 test/test")'); + tool("quint", 'console.log("0.32.0")'); + const environment = { ...process.env, PATH: `${join(temporary, "bin")}${delimiter}${process.env.PATH ?? ""}` }; + const options = { directory: temporary, environment, nodeVersion: "v24.20.0" }; + tool("cargo", 'console.error("cargo: command not found"); process.exit(127)'); + for (const target of ["check-rust", "formal-rust", "smoke", "check"]) expect(() => checkPrerequisites(target, options), target).toThrow(/Cannot run cargo/); + for (const target of ["check-ts", "check-go", "formal-ts", "formal-go", "mutations", "audit"]) expect(() => checkPrerequisites(target, options), target).not.toThrow(); + tool("cargo", 'console.log("cargo 1.97.0 (abcdef 2026-06-01)")'); + expect(() => checkPrerequisites("check-rust", options)).toThrow(/requires cargo 1\.98\.1; found cargo 1\.97\.0/); + tool("cargo", 'console.log("cargo 1.98.10 (abcdef 2026-06-01)")'); + expect(() => checkPrerequisites("formal-rust", options)).toThrow(/requires cargo 1\.98\.1/); + tool("cargo", 'console.log("cargo 1.98.1 (797e8a9bc 2026-08-05)")'); + for (const target of ["check-rust", "formal-rust", "smoke"]) expect(() => checkPrerequisites(target, options), target).not.toThrow(); + } finally { + rmSync(temporary, { recursive: true, force: true }); + } + }); +}); diff --git a/test/formal-validation.test.ts b/test/formal-validation.test.ts index 26db097d..fad5b4e1 100644 --- a/test/formal-validation.test.ts +++ b/test/formal-validation.test.ts @@ -62,6 +62,7 @@ process.exit(Number(process.argv[3] ?? 0));\n`); put("formal/generated-fixtures.lock.json", '{"quintVersion":"0.32.0"}'); fakeTool("corepack", 'console.log("10.33.0")'); fakeTool("go", 'console.log("go version go1.27.1 test/test")'); + fakeTool("cargo", 'console.log("cargo 1.98.1 (test 2026-08-05)")'); fakeTool("quint", 'console.log("0.32.0")'); fakeTool("java", 'console.log("openjdk 21.0.11")'); fakeTool("tar", 'console.log("bsdtar 3.5.3")'); @@ -118,10 +119,10 @@ process.exit(Number(process.argv[3] ?? 0));\n`); expect(plan.filter(step => step.args?.[0] === "formal/witnesses.mjs")).toHaveLength(1); expect(plan.some(step => step.args?.[0] === "formal/generate-artifacts.mjs")).toBe(false); expect(plan[0]!.args).toEqual(["formal/run-models.mjs", "check"]); - expect(plan.find(step => step.remove)!.remove).toEqual([".formal-traces/ts-completion.json", ".formal-traces/go-completion.json"]); + expect(plan.find(step => step.remove)!.remove).toEqual([".formal-traces/ts-completion.json", ".formal-traces/go-completion.json", ".formal-traces/rust-completion.json"]); // The aggregate is exactly these lanes in order, so a CI job running // one lane executes the same steps as the local sequential run. - expect(plan).toEqual(["formal-check", "formal-generate", "formal-ts", "formal-go"].flatMap(target => validationPlan(target, { directory }))); + expect(plan).toEqual(["formal-check", "formal-generate", "formal-ts", "formal-go", "formal-rust"].flatMap(target => validationPlan(target, { directory }))); }); it("keeps the model check as its own lane that produces nothing the port lanes consume", () => { From 5885154b4b30e233c5cc293a60bb65551962777b Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 00:21:49 -0700 Subject: [PATCH 04/31] feat(rust): implement static policy parsing, overlay merge and per-invocation resolution Port go/policy.go into rust/src/policy.rs: Policy::from_json accepts the TypeScript JSON shape and rejects shadowRamp, null leaves and non-object containers; to_json emits Go's staticPolicyMap shape; validate enforces the TTL, ramp, recovery-age and read-deadline domains; resolve_policy merges a sparse runtime overlay once and applies the portable failure scopes (a malformed container, flag or deadline fails the invocation, an invalid TTL or ramp disables only that layer, an invalid recovery or shadow option preserves serving with a diagnostic flag). Tests port every case of go/policy_test.go and add JSON-shape, preset and SPEC overlay-table cases, using only ramps that short-circuit cohort sampling. --- rust/src/policy.rs | 1332 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 1317 insertions(+), 15 deletions(-) diff --git a/rust/src/policy.rs b/rust/src/policy.rs index 3af8fd9a..11e47f13 100644 --- a/rust/src/policy.rs +++ b/rust/src/policy.rs @@ -1,6 +1,9 @@ //! Static operation policy, sparse runtime overlays and per-invocation resolution. +use serde_json::{Map, Value}; + use crate::limits::{DEFAULT_REMOTE_READ_TIMEOUT_MS, MAX_CACHE_TTL_SEC, MAX_DEADLINE_MS}; +use crate::observe::DisabledReason; /// Per-use-case shadow validation policy. #[derive(Debug, Clone, PartialEq, Default)] @@ -55,7 +58,10 @@ impl Policy { stale_on_error_max_age_sec: Some(0), local_ramp: Some(0.0), remote_ramp: Some(0.0), - shadow: Some(ShadowPolicy { ramp: Some(0.0), log_mismatches: Some(false) }), + shadow: Some(ShadowPolicy { + ramp: Some(0.0), + log_mismatches: Some(false), + }), ..Policy::default() } } @@ -108,19 +114,177 @@ impl Policy { /// Parse the JSON-shaped TypeScript configuration /// (`ttlSec`, `ramp`, `shadow`, `requestLocal`, `coalesce`, /// `staleOnErrorMaxAgeSec`, `remoteReadTimeoutMs`). `null` or absent means an empty policy. - pub fn from_json(value: &serde_json::Value) -> Result { - let _ = value; - todo!("Policy::from_json") + /// + /// A present leaf holding `null` is an invalid supplied value, not an + /// omission: JSON has no way to spell JavaScript's `undefined`, so every + /// member of an object counts as present. + pub fn from_json(value: &Value) -> Result { + let mut policy = Policy::default(); + if value.is_null() { + return Ok(policy); + } + let config = policy_object(value, "defaultConfig")?; + reject_shadow_ramp(config)?; + if let Some(layers) = config.get("ttlSec") { + let layers = policy_object(layers, "ttlSec")?; + for (layer, slot) in [ + ("local", &mut policy.local_ttl_sec), + ("remote", &mut policy.remote_ttl_sec), + ] { + if let Some(leaf) = layers.get(layer) { + let seconds = ttl_sec(leaf) + .ok_or_else(|| PolicyError(format!("invalid static ttlSec.{layer}")))?; + *slot = Some(seconds); + } + } + } + if let Some(layers) = config.get("ramp") { + let layers = policy_object(layers, "ramp")?; + for (layer, slot) in [ + ("local", &mut policy.local_ramp), + ("remote", &mut policy.remote_ramp), + ] { + if let Some(leaf) = layers.get(layer) { + let ramp = finite_range(leaf, 0.0, 100.0, false) + .ok_or_else(|| PolicyError(format!("invalid static ramp.{layer}")))?; + *slot = Some(ramp); + } + } + } + for (field, slot) in [ + ("requestLocal", &mut policy.request_local), + ("coalesce", &mut policy.coalesce), + ] { + if let Some(leaf) = config.get(field) { + let flag = leaf + .as_bool() + .ok_or_else(|| PolicyError(format!("{field} must be boolean")))?; + *slot = Some(flag); + } + } + if let Some(leaf) = config.get("staleOnErrorMaxAgeSec") { + let seconds = finite_range(leaf, 0.0, MAX_CACHE_TTL_SEC as f64, true) + .ok_or_else(|| PolicyError("invalid static staleOnErrorMaxAgeSec".to_string()))?; + policy.stale_on_error_max_age_sec = Some(seconds as u64); + } + if let Some(leaf) = config.get("remoteReadTimeoutMs") { + let ms = finite_range(leaf, 1.0, MAX_DEADLINE_MS as f64, true) + .ok_or_else(|| PolicyError("invalid remoteReadTimeoutMs".to_string()))?; + policy.remote_read_timeout_ms = Some(ms as u64); + } + if let Some(shadow) = config.get("shadow") { + let shadow = policy_object(shadow, "shadow")?; + let mut parsed = ShadowPolicy::default(); + if let Some(leaf) = shadow.get("ramp") { + let ramp = finite_range(leaf, 0.0, 100.0, false) + .ok_or_else(|| PolicyError("invalid static shadow.ramp".to_string()))?; + parsed.ramp = Some(ramp); + } + if let Some(leaf) = shadow.get("logMismatches") { + let flag = leaf.as_bool().ok_or_else(|| { + PolicyError("shadow.logMismatches must be boolean".to_string()) + })?; + parsed.log_mismatches = Some(flag); + } + policy.shadow = Some(parsed); + } + policy.validate()?; + Ok(policy) } /// The JSON-shaped form of this policy, with omitted leaves absent. - pub fn to_json(&self) -> serde_json::Value { - todo!("Policy::to_json") + /// + /// `ttlSec` and `ramp` are always objects (possibly empty); `requestLocal` + /// (default false) and `coalesce` (default true) are always present; + /// the optional scalars and the `shadow` object appear only when set. + pub fn to_json(&self) -> Value { + let mut ttl = Map::new(); + if let Some(seconds) = self.local_ttl_sec { + ttl.insert("local".to_string(), Value::from(seconds)); + } + if let Some(seconds) = self.remote_ttl_sec { + ttl.insert("remote".to_string(), Value::from(seconds)); + } + let mut ramp = Map::new(); + if let Some(value) = self.local_ramp { + ramp.insert("local".to_string(), ramp_json(value)); + } + if let Some(value) = self.remote_ramp { + ramp.insert("remote".to_string(), ramp_json(value)); + } + let mut object = Map::new(); + object.insert("ttlSec".to_string(), Value::Object(ttl)); + object.insert("ramp".to_string(), Value::Object(ramp)); + object.insert( + "requestLocal".to_string(), + Value::Bool(self.request_local.unwrap_or(false)), + ); + object.insert( + "coalesce".to_string(), + Value::Bool(self.coalesce.unwrap_or(true)), + ); + if let Some(seconds) = self.stale_on_error_max_age_sec { + object.insert("staleOnErrorMaxAgeSec".to_string(), Value::from(seconds)); + } + if let Some(ms) = self.remote_read_timeout_ms { + object.insert("remoteReadTimeoutMs".to_string(), Value::from(ms)); + } + if let Some(shadow) = &self.shadow { + let mut leaves = Map::new(); + if let Some(value) = shadow.ramp { + leaves.insert("ramp".to_string(), ramp_json(value)); + } + if let Some(flag) = shadow.log_mismatches { + leaves.insert("logMismatches".to_string(), Value::Bool(flag)); + } + object.insert("shadow".to_string(), Value::Object(leaves)); + } + Value::Object(object) } /// Reject statically invalid leaves. pub fn validate(&self) -> Result<(), PolicyError> { - todo!("Policy::validate") + for ttl in [self.local_ttl_sec, self.remote_ttl_sec] + .into_iter() + .flatten() + { + if !(1..=MAX_CACHE_TTL_SEC).contains(&ttl) { + return Err(PolicyError( + "static TTL must be whole seconds within 365 days".to_string(), + )); + } + } + for ramp in [self.local_ramp, self.remote_ramp].into_iter().flatten() { + if !ramp_in_domain(ramp) { + return Err(PolicyError( + "static ramp must be between zero and 100".to_string(), + )); + } + } + if let Some(age) = self.stale_on_error_max_age_sec { + let exceeds_remote = match self.remote_ttl_sec { + Some(remote) if remote > 0 => age > remote, + _ => false, + }; + if age > MAX_CACHE_TTL_SEC || (age > 0 && !exceeds_remote) { + return Err(PolicyError( + "static recovery age must exceed a positive remote TTL".to_string(), + )); + } + } + if let Some(ms) = self.remote_read_timeout_ms { + if !(1..=MAX_DEADLINE_MS).contains(&ms) { + return Err(PolicyError("invalid remote read deadline".to_string())); + } + } + if let Some(ramp) = self.shadow.as_ref().and_then(|shadow| shadow.ramp) { + if !ramp_in_domain(ramp) { + return Err(PolicyError( + "static shadow ramp must be between zero and 100".to_string(), + )); + } + } + Ok(()) } } @@ -131,10 +295,10 @@ impl Policy { /// by the portable contract (an invalid TTL or ramp disables only that /// layer, an invalid flag or read deadline bypasses caching for the call). #[derive(Debug, Clone, PartialEq)] -pub struct RuntimePolicy(pub serde_json::Value); +pub struct RuntimePolicy(pub Value); impl RuntimePolicy { - pub fn from_json(value: serde_json::Value) -> Self { + pub fn from_json(value: Value) -> Self { RuntimePolicy(value) } } @@ -145,8 +309,8 @@ impl From for RuntimePolicy { } } -impl From for RuntimePolicy { - fn from(value: serde_json::Value) -> Self { +impl From for RuntimePolicy { + fn from(value: Value) -> Self { RuntimePolicy(value) } } @@ -159,7 +323,9 @@ pub struct PolicyDefaults { impl Default for PolicyDefaults { fn default() -> Self { - PolicyDefaults { remote_read_timeout_ms: DEFAULT_REMOTE_READ_TIMEOUT_MS } + PolicyDefaults { + remote_read_timeout_ms: DEFAULT_REMOTE_READ_TIMEOUT_MS, + } } } @@ -173,13 +339,25 @@ pub struct PolicyError(pub String); pub struct ResolvedLayer { pub enabled: bool, /// Why the layer is disabled, when it is. - pub reason: Option, + pub reason: Option, /// A valid TTL and ramp remain available when the ramp excluded this key. pub configured: bool, pub ttl_ms: u64, pub ramp: f64, } +impl ResolvedLayer { + fn disabled(reason: DisabledReason) -> Self { + ResolvedLayer { + enabled: false, + reason: Some(reason), + configured: false, + ttl_ms: 0, + ramp: 0.0, + } + } +} + /// Shadow policy after resolution. #[derive(Debug, Clone, Copy, PartialEq, Default)] pub struct ResolvedShadow { @@ -217,6 +395,1130 @@ pub fn resolve_policy( logical_key: &str, defaults: PolicyDefaults, ) -> Result { - let _ = (base, overlay, logical_key, defaults, MAX_CACHE_TTL_SEC, MAX_DEADLINE_MS); - todo!("resolve_policy") + let mut remote_read_timeout_ms = defaults.remote_read_timeout_ms; + if remote_read_timeout_ms == 0 { + remote_read_timeout_ms = DEFAULT_REMOTE_READ_TIMEOUT_MS; + } + if remote_read_timeout_ms > MAX_DEADLINE_MS { + return Err(PolicyError("invalid instance read deadline".to_string())); + } + base.validate()?; + let mut merged = match base.to_json() { + Value::Object(object) => object, + _ => unreachable!("Policy::to_json always produces an object"), + }; + if let Some(RuntimePolicy(overlay)) = overlay { + if !overlay.is_null() { + merge_overlay(&mut merged, overlay)?; + } + } + + let request_local = merged + .get("requestLocal") + .and_then(Value::as_bool) + .ok_or_else(|| PolicyError("runtime requestLocal must be boolean".to_string()))?; + let coalesce = merged + .get("coalesce") + .and_then(Value::as_bool) + .ok_or_else(|| PolicyError("runtime coalesce must be boolean".to_string()))?; + if let Some(leaf) = merged.get("remoteReadTimeoutMs") { + let ms = finite_range(leaf, 1.0, MAX_DEADLINE_MS as f64, true) + .ok_or_else(|| PolicyError("invalid runtime remoteReadTimeoutMs".to_string()))?; + remote_read_timeout_ms = ms as u64; + } + + let empty = Map::new(); + let ttls = merged + .get("ttlSec") + .and_then(Value::as_object) + .unwrap_or(&empty); + let ramps = merged + .get("ramp") + .and_then(Value::as_object) + .unwrap_or(&empty); + let local = resolve_layer(ttls, ramps, logical_key, "local"); + let remote = resolve_layer(ttls, ramps, logical_key, "remote"); + + let mut stale_on_error_max_age_ms = 0; + let mut stale_on_error_config_error = false; + if let Some(age) = merged.get("staleOnErrorMaxAgeSec") { + let numeric_zero = policy_number(age) == Some(0.0); + if !remote.configured { + stale_on_error_config_error = + remote.reason == Some(DisabledReason::PolicyDisabled) && !numeric_zero; + } else if !numeric_zero { + match ttl_sec(age).map(|seconds| seconds * 1_000) { + Some(ms) if ms > remote.ttl_ms => stale_on_error_max_age_ms = ms, + _ => stale_on_error_config_error = true, + } + } + } + + let mut shadow = ResolvedShadow::default(); + if let Some(leaves) = merged.get("shadow").and_then(Value::as_object) { + if let Some(leaf) = leaves.get("ramp") { + match finite_range(leaf, 0.0, 100.0, false) { + Some(ramp) => { + shadow.ramp = ramp; + shadow.enabled = admits(ramp, logical_key, "shadow"); + } + None => shadow.config_error = true, + } + } + if let Some(leaf) = leaves.get("logMismatches") { + match leaf.as_bool() { + Some(flag) => shadow.log_mismatches = flag, + None => { + shadow.log_mismatches = false; + shadow.logging_config_error = true; + } + } + } + } + + Ok(ResolvedPolicy { + request_local, + coalesce, + local, + remote, + remote_read_timeout_ms, + stale_on_error_max_age_ms, + stale_on_error_config_error, + shadow, + }) +} + +/// Apply a sparse runtime overlay to the static policy's JSON form. Every +/// present member counts, including `null`, so an explicit null leaf replaces +/// the operation leaf and is then judged as an invalid supplied value. +fn merge_overlay(merged: &mut Map, overlay: &Value) -> Result<(), PolicyError> { + let overlay = policy_object(overlay, "runtime config")?; + reject_shadow_ramp(overlay)?; + for (name, leaves) in [ + ("ttlSec", &["local", "remote"][..]), + ("ramp", &["local", "remote"][..]), + ("shadow", &["ramp", "logMismatches"][..]), + ] { + let Some(incoming) = overlay.get(name) else { + continue; + }; + let incoming = policy_object(incoming, name)?; + let mut output = match merged.get(name) { + Some(Value::Object(existing)) => existing.clone(), + _ => Map::new(), + }; + for leaf in leaves { + if let Some(value) = incoming.get(*leaf) { + output.insert((*leaf).to_string(), value.clone()); + } + } + merged.insert(name.to_string(), Value::Object(output)); + } + for name in [ + "requestLocal", + "coalesce", + "staleOnErrorMaxAgeSec", + "remoteReadTimeoutMs", + ] { + if let Some(value) = overlay.get(name) { + merged.insert(name.to_string(), value.clone()); + } + } + Ok(()) +} + +fn resolve_layer( + ttls: &Map, + ramps: &Map, + logical_key: &str, + layer: &str, +) -> ResolvedLayer { + let Some(ttl) = ttls.get(layer) else { + return ResolvedLayer::disabled(DisabledReason::PolicyDisabled); + }; + let Some(ttl_ms) = ttl_sec(ttl).map(|seconds| seconds * 1_000) else { + return ResolvedLayer::disabled(DisabledReason::InvalidTtl); + }; + let ramp = match ramps.get(layer) { + None => 100.0, + Some(leaf) => match finite_range(leaf, 0.0, 100.0, false) { + Some(ramp) => ramp, + None => return ResolvedLayer::disabled(DisabledReason::InvalidRamp), + }, + }; + let enabled = admits(ramp, logical_key, layer); + ResolvedLayer { + enabled, + reason: if enabled { + None + } else { + Some(DisabledReason::RampedDown) + }, + configured: true, + ttl_ms, + ramp, + } +} + +/// Cohort admission: a full ramp admits every key, a zero ramp none, and a +/// partial ramp admits keys whose stable sample is strictly below it. +fn admits(ramp: f64, logical_key: &str, discriminator: &str) -> bool { + ramp >= 100.0 || (ramp > 0.0 && crate::identity::cohort(logical_key, discriminator) < ramp) +} + +fn ramp_in_domain(ramp: f64) -> bool { + ramp.is_finite() && (0.0..=100.0).contains(&ramp) +} + +/// Spell a ramp the way JavaScript and Go's `encoding/json` do: an integral +/// value has no fraction (`100`, not `100.0`), so a static policy's JSON form +/// compares equal to the same policy parsed from text. +fn ramp_json(ramp: f64) -> Value { + if ramp_in_domain(ramp) && ramp.trunc() == ramp { + Value::from(ramp as u64) + } else { + Value::from(ramp) + } +} + +/// The numeric domain: JSON numbers only. Strings and booleans never count. +fn policy_number(value: &Value) -> Option { + value.as_number().and_then(serde_json::Number::as_f64) +} + +fn finite_range(value: &Value, min: f64, max: f64, integer: bool) -> Option { + let n = policy_number(value)?; + (n.is_finite() && n >= min && n <= max && (!integer || n.trunc() == n)).then_some(n) +} + +/// A TTL-domain leaf in whole seconds. +fn ttl_sec(value: &Value) -> Option { + finite_range(value, 1.0, MAX_CACHE_TTL_SEC as f64, true).map(|n| n as u64) +} + +fn policy_object<'a>(value: &'a Value, name: &str) -> Result<&'a Map, PolicyError> { + value + .as_object() + .ok_or_else(|| PolicyError(format!("DialCache {name} must be an object"))) +} + +fn reject_shadow_ramp(object: &Map) -> Result<(), PolicyError> { + if object.contains_key("shadowRamp") { + return Err(PolicyError( + "shadowRamp was replaced by shadow.ramp".to_string(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + const KEY: &str = "policy:item:one#lookup"; + const MAX_TTL: u64 = MAX_CACHE_TTL_SEC; + + fn resolve(base: &Policy, overlay: Option) -> Result { + let overlay = overlay.map(RuntimePolicy); + resolve_policy(base, overlay.as_ref(), KEY, PolicyDefaults::default()) + } + + fn resolved(base: &Policy, overlay: Option) -> ResolvedPolicy { + resolve(base, overlay).expect("policy resolves") + } + + // Port of TestPolicySparseResolutionAndSnapshots. + #[test] + fn sparse_resolution_and_inheritance() { + let base = Policy::from_json(&json!({ + "requestLocal": true, "coalesce": false, + "ttlSec": {"local": 1.0, "remote": 2.0}, + "staleOnErrorMaxAgeSec": 5.0, "remoteReadTimeoutMs": 30.0, + "shadow": {"ramp": 100.0, "logMismatches": true}, + })) + .unwrap(); + let overlay = RuntimePolicy(json!({ + "ttlSec": {"remote": 4.0}, + "ramp": {"local": 0.0}, + "shadow": {"logMismatches": false}, + })); + let r = resolve_policy( + &base, + Some(&overlay), + KEY, + PolicyDefaults { + remote_read_timeout_ms: 20, + }, + ) + .unwrap(); + assert!(r.request_local); + assert!(!r.coalesce); + assert!(!r.local.enabled); + assert_eq!(r.local.reason, Some(DisabledReason::RampedDown)); + assert!(r.local.configured); + assert_eq!(r.local.ttl_ms, 1_000); + assert_eq!(r.local.ramp, 0.0); + assert!(r.remote.enabled); + assert_eq!(r.remote.reason, None); + assert_eq!(r.remote.ttl_ms, 4_000); + assert_eq!(r.remote.ramp, 100.0); + assert_eq!(r.stale_on_error_max_age_ms, 5_000); + assert!(!r.stale_on_error_config_error); + assert_eq!(r.remote_read_timeout_ms, 30); + assert!(r.shadow.enabled); + assert_eq!(r.shadow.ramp, 100.0); + assert!(!r.shadow.log_mismatches); + assert!(!r.shadow.config_error); + assert!(!r.shadow.logging_config_error); + + // A clone is an independent snapshot of every leaf. + let snapshot = base.clone(); + let mut mutated = base.clone(); + mutated.remote_read_timeout_ms = Some(99); + mutated.shadow.as_mut().unwrap().ramp = Some(0.0); + assert_eq!(snapshot.remote_read_timeout_ms, Some(30)); + assert_eq!(snapshot.shadow.as_ref().unwrap().ramp, Some(100.0)); + + // A null provider reply inherits the whole operation policy. + for overlay in [None, Some(Value::Null)] { + let inherit = resolve_policy( + &snapshot, + overlay.map(RuntimePolicy).as_ref(), + KEY, + PolicyDefaults::default(), + ) + .unwrap(); + assert!(inherit.local.enabled); + assert!(inherit.remote.enabled); + assert_eq!(inherit.remote_read_timeout_ms, 30); + assert!(inherit.shadow.enabled); + assert!(inherit.shadow.log_mismatches); + } + } + + // Port of TestPolicyStaticValidation. + #[test] + fn static_validation_rejects_malformed_configs() { + let invalid = [ + json!(true), + json!([]), + json!("policy"), + json!(1), + json!({"shadowRamp": 1}), + json!({"ttlSec": null}), + json!({"ramp": null}), + json!({"shadow": null}), + json!({"ttlSec": []}), + json!({"ttlSec": 60}), + json!({"shadow": true}), + json!({"requestLocal": null}), + json!({"coalesce": null}), + json!({"coalesce": "false"}), + json!({"requestLocal": 1}), + json!({"ttlSec": {"local": null}}), + json!({"ttlSec": {"local": 0}}), + json!({"ttlSec": {"local": -1}}), + json!({"ttlSec": {"local": 1.5}}), + json!({"ttlSec": {"local": "60"}}), + json!({"ttlSec": {"remote": 31_536_001}}), + json!({"ramp": {"remote": null}}), + json!({"ramp": {"remote": "nan"}}), + json!({"ramp": {"remote": -0.5}}), + json!({"ramp": {"local": 100.5}}), + json!({"ramp": {"local": true}}), + json!({"remoteReadTimeoutMs": null}), + json!({"remoteReadTimeoutMs": 0}), + json!({"remoteReadTimeoutMs": 1.5}), + json!({"remoteReadTimeoutMs": 2_147_483_648u64}), + json!({"staleOnErrorMaxAgeSec": null}), + json!({"staleOnErrorMaxAgeSec": -1}), + json!({"staleOnErrorMaxAgeSec": 2.5}), + json!({"staleOnErrorMaxAgeSec": 2}), + json!({"ttlSec": {"remote": 2}, "staleOnErrorMaxAgeSec": 2}), + json!({"ttlSec": {"remote": 2}, "staleOnErrorMaxAgeSec": 1}), + json!({"ttlSec": {"local": 2}, "staleOnErrorMaxAgeSec": 3}), + json!({"ttlSec": {"remote": 2}, "staleOnErrorMaxAgeSec": 31_536_001}), + json!({"shadow": {"ramp": null}}), + json!({"shadow": {"ramp": 101}}), + json!({"shadow": {"ramp": "50"}}), + json!({"shadow": {"logMismatches": null}}), + json!({"shadow": {"logMismatches": 1}}), + ]; + for config in invalid { + assert!( + Policy::from_json(&config).is_err(), + "accepted invalid static config: {config}" + ); + } + + let accepted = Policy::from_json(&json!({ + "ttlSec": {"remote": 31_536_000}, + "remoteReadTimeoutMs": 2_147_483_647u64, + })) + .unwrap(); + assert_eq!(accepted.remote_ttl_sec, Some(MAX_TTL)); + assert_eq!(accepted.remote_read_timeout_ms, Some(MAX_DEADLINE_MS)); + assert_eq!(accepted.local_ttl_sec, None); + + // Zero recovery age is valid without a remote TTL; a positive one needs to exceed it. + assert!(Policy::from_json(&json!({"staleOnErrorMaxAgeSec": 0})).is_ok()); + assert!( + Policy::from_json(&json!({"ttlSec": {"remote": 2}, "staleOnErrorMaxAgeSec": 3})) + .is_ok() + ); + // Integer-valued floats and unsigned integers share one numeric domain. + let floats = + Policy::from_json(&json!({"ttlSec": {"local": 60.0}, "ramp": {"local": 50}})).unwrap(); + assert_eq!(floats.local_ttl_sec, Some(60)); + assert_eq!(floats.local_ramp, Some(50.0)); + // Ramp boundaries are inclusive. + assert!(Policy::from_json( + &json!({"ramp": {"local": 0, "remote": 100}, "shadow": {"ramp": 0}}) + ) + .is_ok()); + + // Direct validation mirrors the same domains without the parser. + assert!(Policy::default().local_ttl_sec(0).validate().is_err()); + assert!(Policy::default() + .local_ttl_sec(MAX_TTL + 1) + .validate() + .is_err()); + assert!(Policy::default().remote_ttl_sec(MAX_TTL).validate().is_ok()); + assert!(Policy::default().local_ramp(f64::NAN).validate().is_err()); + assert!(Policy::default() + .remote_ramp(f64::INFINITY) + .validate() + .is_err()); + assert!(Policy::default().remote_ramp(100.0001).validate().is_err()); + assert!(Policy::default().local_ramp(-0.1).validate().is_err()); + assert!(Policy::default() + .remote_read_timeout_ms(0) + .validate() + .is_err()); + assert!(Policy::default() + .remote_read_timeout_ms(MAX_DEADLINE_MS + 1) + .validate() + .is_err()); + assert!(Policy::default() + .remote_read_timeout_ms(MAX_DEADLINE_MS) + .validate() + .is_ok()); + assert!(Policy::default() + .stale_on_error_max_age_sec(1) + .validate() + .is_err()); + assert!(Policy::default() + .stale_on_error_max_age_sec(0) + .validate() + .is_ok()); + assert!(Policy::default() + .remote_ttl_sec(2) + .stale_on_error_max_age_sec(2) + .validate() + .is_err()); + assert!(Policy::default() + .remote_ttl_sec(2) + .stale_on_error_max_age_sec(3) + .validate() + .is_ok()); + assert!(Policy::default() + .remote_ttl_sec(2) + .stale_on_error_max_age_sec(MAX_TTL + 1) + .validate() + .is_err()); + let nan_shadow = ShadowPolicy { + ramp: Some(f64::NAN), + log_mismatches: None, + }; + assert!(Policy::default().shadow(nan_shadow).validate().is_err()); + let high_shadow = ShadowPolicy { + ramp: Some(100.5), + log_mismatches: None, + }; + assert!(Policy::default().shadow(high_shadow).validate().is_err()); + let flag_only = ShadowPolicy { + ramp: None, + log_mismatches: Some(true), + }; + assert!(Policy::default().shadow(flag_only).validate().is_ok()); + assert!(Policy::enabled(60).validate().is_ok()); + assert!(Policy::disabled().validate().is_ok()); + } + + #[test] + fn from_json_treats_absent_and_null_wholes_as_empty() { + assert_eq!(Policy::from_json(&Value::Null).unwrap(), Policy::default()); + assert_eq!(Policy::from_json(&json!({})).unwrap(), Policy::default()); + // Unknown members are ignored, as in the TypeScript constructor. + assert_eq!( + Policy::from_json(&json!({"unknown": 1})).unwrap(), + Policy::default() + ); + // Empty containers are valid and leave every leaf omitted. + let empty = Policy::from_json(&json!({"ttlSec": {}, "ramp": {}, "shadow": {}})).unwrap(); + assert_eq!(empty.local_ttl_sec, None); + assert_eq!(empty.shadow, Some(ShadowPolicy::default())); + } + + #[test] + fn from_json_round_trips_through_to_json() { + let source = json!({ + "ttlSec": {"local": 5, "remote": 60}, + "ramp": {"local": 12.5, "remote": 100}, + "requestLocal": true, + "coalesce": false, + "staleOnErrorMaxAgeSec": 120, + "remoteReadTimeoutMs": 25, + "shadow": {"ramp": 3.5, "logMismatches": true}, + }); + let policy = Policy::from_json(&source).unwrap(); + assert_eq!( + policy, + Policy { + request_local: Some(true), + coalesce: Some(false), + local_ttl_sec: Some(5), + remote_ttl_sec: Some(60), + local_ramp: Some(12.5), + remote_ramp: Some(100.0), + stale_on_error_max_age_sec: Some(120), + remote_read_timeout_ms: Some(25), + shadow: Some(ShadowPolicy { + ramp: Some(3.5), + log_mismatches: Some(true) + }), + } + ); + assert_eq!(policy.to_json(), source); + assert_eq!(Policy::from_json(&policy.to_json()).unwrap(), policy); + } + + #[test] + fn to_json_matches_the_static_policy_map_shape() { + // Empty policy: layer maps present but empty, flags at their defaults, optionals absent. + assert_eq!( + Policy::default().to_json(), + json!({"ttlSec": {}, "ramp": {}, "requestLocal": false, "coalesce": true}) + ); + // Only present leaves appear inside the layer maps; integral ramps spell without a fraction. + assert_eq!( + Policy::default() + .local_ttl_sec(7) + .remote_ramp(40.0) + .to_json(), + json!({"ttlSec": {"local": 7}, "ramp": {"remote": 40}, "requestLocal": false, "coalesce": true}) + ); + assert_eq!( + Policy::default().local_ramp(12.5).to_json(), + json!({"ttlSec": {}, "ramp": {"local": 12.5}, "requestLocal": false, "coalesce": true}) + ); + // An explicit flag value is emitted even when it equals the default. + assert_eq!( + Policy::default() + .request_local(false) + .coalesce(true) + .to_json(), + json!({"ttlSec": {}, "ramp": {}, "requestLocal": false, "coalesce": true}) + ); + // A present but empty shadow policy is an empty object. + assert_eq!( + Policy::default().shadow(ShadowPolicy::default()).to_json(), + json!({"ttlSec": {}, "ramp": {}, "requestLocal": false, "coalesce": true, "shadow": {}}) + ); + assert_eq!( + Policy::default() + .shadow(ShadowPolicy { + ramp: None, + log_mismatches: Some(false) + }) + .to_json(), + json!({ + "ttlSec": {}, "ramp": {}, "requestLocal": false, "coalesce": true, + "shadow": {"logMismatches": false}, + }) + ); + // Zero recovery age is a present leaf, not an omission. + assert_eq!( + Policy::default() + .stale_on_error_max_age_sec(0) + .remote_read_timeout_ms(75) + .to_json(), + json!({ + "ttlSec": {}, "ramp": {}, "requestLocal": false, "coalesce": true, + "staleOnErrorMaxAgeSec": 0, "remoteReadTimeoutMs": 75, + }) + ); + assert_eq!( + Policy::enabled(60).to_json(), + json!({ + "ttlSec": {"local": 60, "remote": 60}, + "ramp": {"local": 100, "remote": 100}, + "requestLocal": false, "coalesce": true, + }) + ); + assert_eq!( + Policy::disabled().to_json(), + json!({ + "ttlSec": {}, + "ramp": {"local": 0, "remote": 0}, + "requestLocal": false, "coalesce": true, + "staleOnErrorMaxAgeSec": 0, + "shadow": {"ramp": 0, "logMismatches": false}, + }) + ); + } + + #[test] + fn enabled_and_disabled_presets_resolve_as_documented() { + let enabled = resolved(&Policy::enabled(60), None); + assert!(!enabled.request_local); + assert!(enabled.coalesce); + assert!(enabled.local.enabled && enabled.remote.enabled); + assert_eq!( + (enabled.local.ttl_ms, enabled.remote.ttl_ms), + (60_000, 60_000) + ); + assert_eq!((enabled.local.ramp, enabled.remote.ramp), (100.0, 100.0)); + assert_eq!( + enabled.remote_read_timeout_ms, + DEFAULT_REMOTE_READ_TIMEOUT_MS + ); + assert_eq!(enabled.stale_on_error_max_age_ms, 0); + assert!(!enabled.stale_on_error_config_error); + assert_eq!(enabled.shadow, ResolvedShadow::default()); + + // Standing alone, the kill switch has no TTLs: both layers are simply not configured, + // and its zero recovery age is not a configuration error. + let alone = resolved(&Policy::disabled(), None); + assert_eq!(alone.local.reason, Some(DisabledReason::PolicyDisabled)); + assert_eq!(alone.remote.reason, Some(DisabledReason::PolicyDisabled)); + assert!(!alone.stale_on_error_config_error); + assert!(!alone.shadow.enabled && !alone.shadow.config_error); + assert!(!alone.shadow.log_mismatches && !alone.shadow.logging_config_error); + + // As an overlay it ramps every inherited path to zero instead of relying on omission. + let base = Policy::enabled(60) + .request_local(true) + .stale_on_error_max_age_sec(120) + .shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: Some(true), + }); + let killed = resolved(&base, Some(RuntimePolicy::from(Policy::disabled()).0)); + assert!(!killed.request_local); + assert!(killed.coalesce); + for layer in [killed.local, killed.remote] { + assert!(!layer.enabled); + assert_eq!(layer.reason, Some(DisabledReason::RampedDown)); + assert!(layer.configured); + assert_eq!(layer.ttl_ms, 60_000); + assert_eq!(layer.ramp, 0.0); + } + assert_eq!(killed.stale_on_error_max_age_ms, 0); + assert!(!killed.stale_on_error_config_error); + assert_eq!(killed.shadow, ResolvedShadow::default()); + } + + // Port of TestRuntimePolicyFailureScopes. + #[test] + fn runtime_failure_scopes() { + let base = Policy::default() + .request_local(true) + .local_ttl_sec(1) + .remote_ttl_sec(2); + for overlay in [ + json!(false), + json!([]), + json!("config"), + json!(0), + json!({"shadowRamp": 5}), + json!({"ttlSec": null}), + json!({"ttlSec": []}), + json!({"ttlSec": 5}), + json!({"ramp": null}), + json!({"shadow": null}), + json!({"shadow": "off"}), + json!({"requestLocal": null}), + json!({"requestLocal": "true"}), + json!({"coalesce": 1}), + json!({"coalesce": null}), + json!({"remoteReadTimeoutMs": 0}), + json!({"remoteReadTimeoutMs": null}), + json!({"remoteReadTimeoutMs": 1.5}), + json!({"remoteReadTimeoutMs": "30"}), + json!({"remoteReadTimeoutMs": 2_147_483_648u64}), + ] { + assert!( + resolve(&base, Some(overlay.clone())).is_err(), + "accepted invalid invocation policy: {overlay}" + ); + } + + struct Case { + overlay: Value, + local: Option, + remote: Option, + recovery_error: bool, + } + let invalid_ttl = Some(DisabledReason::InvalidTtl); + let invalid_ramp = Some(DisabledReason::InvalidRamp); + let cases = [ + Case { + overlay: json!({"ttlSec": {"local": null}}), + local: invalid_ttl, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ttlSec": {"local": "1"}}), + local: invalid_ttl, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ttlSec": {"local": 0}}), + local: invalid_ttl, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ttlSec": {"local": 1.5}}), + local: invalid_ttl, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ttlSec": {"local": 31_536_001}}), + local: invalid_ttl, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ramp": {"remote": true}}), + local: None, + remote: invalid_ramp, + recovery_error: false, + }, + Case { + overlay: json!({"ramp": {"remote": null}}), + local: None, + remote: invalid_ramp, + recovery_error: false, + }, + Case { + overlay: json!({"ramp": {"remote": 100.1}}), + local: None, + remote: invalid_ramp, + recovery_error: false, + }, + Case { + overlay: json!({"ramp": {"local": -1}}), + local: invalid_ramp, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": null}), + local: None, + remote: None, + recovery_error: true, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 1}), + local: None, + remote: None, + recovery_error: true, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 2}), + local: None, + remote: None, + recovery_error: true, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": "3"}), + local: None, + remote: None, + recovery_error: true, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 2.5}), + local: None, + remote: None, + recovery_error: true, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 0}), + local: None, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 0.0}), + local: None, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"staleOnErrorMaxAgeSec": 3}), + local: None, + remote: None, + recovery_error: false, + }, + Case { + overlay: json!({"ttlSec": {"remote": -1}, "staleOnErrorMaxAgeSec": -1}), + local: None, + remote: invalid_ttl, + recovery_error: false, + }, + Case { + overlay: json!({"ramp": {"remote": "bad"}, "staleOnErrorMaxAgeSec": 5}), + local: None, + remote: invalid_ramp, + recovery_error: false, + }, + ]; + for case in cases { + let r = resolved(&base, Some(case.overlay.clone())); + assert!(r.request_local, "{}", case.overlay); + assert!(r.coalesce, "{}", case.overlay); + assert_eq!( + r.local.reason, case.local, + "local reason for {}", + case.overlay + ); + assert_eq!( + r.remote.reason, case.remote, + "remote reason for {}", + case.overlay + ); + assert_eq!(r.local.enabled, case.local.is_none(), "{}", case.overlay); + assert_eq!(r.remote.enabled, case.remote.is_none(), "{}", case.overlay); + assert_eq!( + r.stale_on_error_config_error, case.recovery_error, + "recovery for {}", + case.overlay + ); + if case.recovery_error { + assert_eq!(r.stale_on_error_max_age_ms, 0, "{}", case.overlay); + } + } + // A valid runtime age above the remote TTL enables recovery. + assert_eq!( + resolved(&base, Some(json!({"staleOnErrorMaxAgeSec": 3}))).stale_on_error_max_age_ms, + 3_000 + ); + + // Recovery configured without any remote TTL is a diagnostic error on a disabled layer. + let missing = resolved( + &Policy::default(), + Some(json!({"staleOnErrorMaxAgeSec": 3})), + ); + assert!(missing.stale_on_error_config_error); + assert_eq!(missing.remote.reason, Some(DisabledReason::PolicyDisabled)); + assert!(!missing.remote.configured); + assert_eq!(missing.stale_on_error_max_age_ms, 0); + // ... unless it is numeric zero. + let zero = resolved( + &Policy::default(), + Some(json!({"staleOnErrorMaxAgeSec": 0})), + ); + assert!(!zero.stale_on_error_config_error); + // A non-numeric leaf on a disabled remote layer is still an error. + let text = resolved( + &Policy::default(), + Some(json!({"staleOnErrorMaxAgeSec": "0"})), + ); + assert!(text.stale_on_error_config_error); + } + + // Port of TestRecoveryRetentionAndShadowDiagnosticsRemainIndependent. + #[test] + fn recovery_retention_and_shadow_diagnostics_remain_independent() { + let base = Policy::default() + .remote_ttl_sec(1) + .remote_ramp(0.0) + .stale_on_error_max_age_sec(86_400); + let r = resolved( + &base, + Some(json!({"shadow": {"ramp": 100, "logMismatches": "invalid"}})), + ); + assert!(!r.remote.enabled); + assert!(r.remote.configured); + assert_eq!(r.remote.reason, Some(DisabledReason::RampedDown)); + assert_eq!(r.stale_on_error_max_age_ms, 86_400_000); + assert!(!r.stale_on_error_config_error); + assert!(r.shadow.enabled); + assert_eq!(r.shadow.ramp, 100.0); + assert!(!r.shadow.config_error); + assert!(r.shadow.logging_config_error); + assert!(!r.shadow.log_mismatches); + + let r = resolved(&base, Some(json!({"shadow": {"ramp": null}}))); + assert!(r.shadow.config_error); + assert!(!r.shadow.enabled); + assert_eq!(r.shadow.ramp, 0.0); + assert!(r.remote.configured); + assert_eq!(r.stale_on_error_max_age_ms, 86_400_000); + + for invalid in [json!("50"), json!(-1), json!(100.5), json!(true)] { + let r = resolved(&base, Some(json!({"shadow": {"ramp": invalid}}))); + assert!(r.shadow.config_error, "{invalid}"); + assert!(!r.shadow.enabled, "{invalid}"); + } + // Zero shadow ramp is valid and silently off. + let off = resolved(&base, Some(json!({"shadow": {"ramp": 0}}))); + assert!(!off.shadow.enabled && !off.shadow.config_error); + // A null logging flag is malformed and disables warnings without touching the cohort. + let logging = resolved( + &base, + Some(json!({"shadow": {"ramp": 100, "logMismatches": null}})), + ); + assert!( + logging.shadow.enabled + && logging.shadow.logging_config_error + && !logging.shadow.log_mismatches + ); + } + + #[test] + fn overlay_semantics_follow_the_portable_table() { + let base = Policy::enabled(10) + .request_local(true) + .coalesce(false) + .stale_on_error_max_age_sec(30) + .remote_read_timeout_ms(40) + .shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: Some(true), + }); + + // Omitted leaves inherit the operation leaf; an empty overlay changes nothing. + let inherited = resolved(&base, None); + assert_eq!(resolved(&base, Some(json!({}))), inherited); + assert_eq!( + resolved(&base, Some(json!({"ttlSec": {}, "ramp": {}, "shadow": {}}))), + inherited + ); + assert_eq!(resolved(&base, Some(Value::Null)), inherited); + assert!(inherited.request_local && !inherited.coalesce); + assert_eq!(inherited.remote_read_timeout_ms, 40); + assert_eq!(inherited.stale_on_error_max_age_ms, 30_000); + assert!(inherited.shadow.enabled && inherited.shadow.log_mismatches); + + // A present leaf replaces the operation leaf, sparsely. + let patched = resolved( + &base, + Some(json!({"ttlSec": {"local": 3}, "ramp": {"remote": 0}})), + ); + assert_eq!(patched.local.ttl_ms, 3_000); + assert!(patched.local.enabled); + assert_eq!(patched.remote.ttl_ms, 10_000); + assert_eq!(patched.remote.reason, Some(DisabledReason::RampedDown)); + assert!(patched.remote.configured); + // Recovery still measures against the merged remote TTL. + assert_eq!(patched.stale_on_error_max_age_ms, 30_000); + + // An explicit null leaf is an invalid supplied value that disables only that layer. + let nulled = resolved(&base, Some(json!({"ttlSec": {"remote": null}}))); + assert!(nulled.local.enabled); + assert_eq!(nulled.remote.reason, Some(DisabledReason::InvalidTtl)); + assert!(!nulled.remote.configured); + // The pre-existing recovery age on a layer disabled by an invalid TTL is not a config error. + assert!(!nulled.stale_on_error_config_error); + assert_eq!(nulled.stale_on_error_max_age_ms, 0); + + // An invalid flag fails the whole invocation. + assert!(resolve(&base, Some(json!({"coalesce": "yes"}))).is_err()); + assert!(resolve(&base, Some(json!({"coalesce": null}))).is_err()); + assert!(resolve(&base, Some(json!({"requestLocal": 0}))).is_err()); + // ... as does an invalid runtime read deadline. + assert!(resolve(&base, Some(json!({"remoteReadTimeoutMs": -5}))).is_err()); + // A valid runtime flag or deadline wins over the operation. + let flipped = resolved( + &base, + Some(json!({"coalesce": true, "requestLocal": false, "remoteReadTimeoutMs": 7})), + ); + assert!(flipped.coalesce && !flipped.request_local); + assert_eq!(flipped.remote_read_timeout_ms, 7); + + // An invalid optional recovery age preserves remote serving with a diagnostic error. + let recovery = resolved(&base, Some(json!({"staleOnErrorMaxAgeSec": 10}))); + assert!(recovery.remote.enabled); + assert_eq!(recovery.stale_on_error_max_age_ms, 0); + assert!(recovery.stale_on_error_config_error); + let recovery = resolved(&base, Some(json!({"staleOnErrorMaxAgeSec": "30"}))); + assert!(recovery.remote.enabled && recovery.stale_on_error_config_error); + // Zero disables inherited recovery silently. + let off = resolved(&base, Some(json!({"staleOnErrorMaxAgeSec": 0}))); + assert!(off.remote.enabled); + assert_eq!(off.stale_on_error_max_age_ms, 0); + assert!(!off.stale_on_error_config_error); + // Raising the remote TTL above the inherited age invalidates the inherited recovery. + let raised = resolved(&base, Some(json!({"ttlSec": {"remote": 30}}))); + assert!(raised.remote.enabled && raised.stale_on_error_config_error); + assert_eq!(raised.stale_on_error_max_age_ms, 0); + + // Shadow leaves merge independently of serving leaves. + let shadow = resolved(&base, Some(json!({"shadow": {"logMismatches": false}}))); + assert!(shadow.shadow.enabled && !shadow.shadow.log_mismatches); + let shadow = resolved(&base, Some(json!({"shadow": {"ramp": 0}}))); + assert!( + !shadow.shadow.enabled && shadow.shadow.log_mismatches && !shadow.shadow.config_error + ); + assert!(shadow.local.enabled && shadow.remote.enabled); + } + + #[test] + fn runtime_leaves_can_configure_paths_the_operation_omitted() { + // Runtime TTLs enable layers the operation left off, with an implied full ramp. + let r = resolved( + &Policy::default(), + Some(json!({"ttlSec": {"local": 5, "remote": 6}})), + ); + assert!(r.local.enabled && r.remote.enabled); + assert_eq!((r.local.ramp, r.remote.ramp), (100.0, 100.0)); + assert_eq!((r.local.ttl_ms, r.remote.ttl_ms), (5_000, 6_000)); + // A ramp without a TTL leaves the layer policy-disabled. + let r = resolved(&Policy::default(), Some(json!({"ramp": {"local": 100}}))); + assert_eq!(r.local.reason, Some(DisabledReason::PolicyDisabled)); + assert!(!r.local.configured); + // Runtime shadow and recovery leaves attach to a runtime remote layer. + let r = resolved( + &Policy::default(), + Some( + json!({"ttlSec": {"remote": 6}, "staleOnErrorMaxAgeSec": 7, "shadow": {"ramp": 100}}), + ), + ); + assert_eq!(r.stale_on_error_max_age_ms, 7_000); + assert!(r.shadow.enabled && !r.shadow.log_mismatches); + // A runtime static-policy overlay (via From) behaves like its JSON form. + let overlay = RuntimePolicy::from(Policy::default().local_ttl_sec(9)); + let r = resolve_policy( + &Policy::default(), + Some(&overlay), + KEY, + PolicyDefaults::default(), + ) + .unwrap(); + assert!(r.local.enabled); + assert_eq!(r.local.ttl_ms, 9_000); + assert_eq!(r.remote.reason, Some(DisabledReason::PolicyDisabled)); + } + + #[test] + fn instance_defaults_and_deadline_precedence() { + let base = Policy::enabled(1); + // Zero instance default means the library default. + let r = resolve_policy( + &base, + None, + KEY, + PolicyDefaults { + remote_read_timeout_ms: 0, + }, + ) + .unwrap(); + assert_eq!(r.remote_read_timeout_ms, DEFAULT_REMOTE_READ_TIMEOUT_MS); + // The instance default applies when neither operation nor runtime supplies a budget. + let r = resolve_policy( + &base, + None, + KEY, + PolicyDefaults { + remote_read_timeout_ms: 20, + }, + ) + .unwrap(); + assert_eq!(r.remote_read_timeout_ms, 20); + // Operation beats instance; runtime beats operation. + let op = base.clone().remote_read_timeout_ms(30); + let r = resolve_policy( + &op, + None, + KEY, + PolicyDefaults { + remote_read_timeout_ms: 20, + }, + ) + .unwrap(); + assert_eq!(r.remote_read_timeout_ms, 30); + let overlay = RuntimePolicy(json!({"remoteReadTimeoutMs": 40})); + let r = resolve_policy( + &op, + Some(&overlay), + KEY, + PolicyDefaults { + remote_read_timeout_ms: 20, + }, + ) + .unwrap(); + assert_eq!(r.remote_read_timeout_ms, 40); + // An out-of-domain instance default fails resolution before anything else. + let err = resolve_policy( + &base, + None, + KEY, + PolicyDefaults { + remote_read_timeout_ms: MAX_DEADLINE_MS + 1, + }, + ); + assert_eq!( + err.unwrap_err(), + PolicyError("invalid instance read deadline".to_string()) + ); + assert!(resolve_policy( + &base, + None, + KEY, + PolicyDefaults { + remote_read_timeout_ms: MAX_DEADLINE_MS + } + ) + .is_ok()); + // A statically invalid base fails resolution even with a valid overlay. + let invalid = Policy::default().local_ttl_sec(0); + assert!(resolve(&invalid, Some(json!({"ttlSec": {"local": 5}}))).is_err()); + } + + #[test] + fn error_messages_name_the_offending_container() { + assert_eq!( + Policy::from_json(&json!([])).unwrap_err(), + PolicyError("DialCache defaultConfig must be an object".to_string()) + ); + assert_eq!( + Policy::from_json(&json!({"ttlSec": null})).unwrap_err(), + PolicyError("DialCache ttlSec must be an object".to_string()) + ); + assert_eq!( + Policy::from_json(&json!({"shadowRamp": 1})).unwrap_err(), + PolicyError("shadowRamp was replaced by shadow.ramp".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!(1))).unwrap_err(), + PolicyError("DialCache runtime config must be an object".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!({"shadow": 1}))).unwrap_err(), + PolicyError("DialCache shadow must be an object".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!({"shadowRamp": null}))).unwrap_err(), + PolicyError("shadowRamp was replaced by shadow.ramp".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!({"coalesce": null}))).unwrap_err(), + PolicyError("runtime coalesce must be boolean".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!({"requestLocal": "x"}))).unwrap_err(), + PolicyError("runtime requestLocal must be boolean".to_string()) + ); + assert_eq!( + resolve(&Policy::default(), Some(json!({"remoteReadTimeoutMs": 0}))).unwrap_err(), + PolicyError("invalid runtime remoteReadTimeoutMs".to_string()) + ); + } } From e1b3407a9ffd5f7082c6e66561f5763589717183 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 00:25:17 -0700 Subject: [PATCH 05/31] feat(rust): cache engine with scopes, coalescing, deadlines, recovery and shadow validation --- rust/src/cancel.rs | 18 +- rust/src/clock.rs | 4 +- rust/src/codec.rs | 10 +- rust/src/deadline.rs | 59 ++ rust/src/engine.rs | 687 +++++++++++++++++++++++ rust/src/error.rs | 8 +- rust/src/execution.rs | 994 ++++++++++++++++++++++++++++++++++ rust/src/flight.rs | 167 ++++++ rust/src/identity.rs | 6 +- rust/src/lib.rs | 24 +- rust/src/local.rs | 22 +- rust/src/observe.rs | 100 +++- rust/src/operation.rs | 244 +++++++++ rust/src/policy.rs | 18 +- rust/src/preview.rs | 46 ++ rust/src/protocol/envelope.rs | 14 +- rust/src/protocol/frame.rs | 6 +- rust/src/protocol/mod.rs | 9 +- rust/src/remote.rs | 11 +- rust/src/scope.rs | 104 ++++ rust/src/shadow.rs | 439 +++++++++++++++ rust/src/use_case.rs | 438 +++++++++++++++ 22 files changed, 3380 insertions(+), 48 deletions(-) create mode 100644 rust/src/deadline.rs create mode 100644 rust/src/engine.rs create mode 100644 rust/src/execution.rs create mode 100644 rust/src/flight.rs create mode 100644 rust/src/operation.rs create mode 100644 rust/src/preview.rs create mode 100644 rust/src/scope.rs create mode 100644 rust/src/shadow.rs create mode 100644 rust/src/use_case.rs diff --git a/rust/src/cancel.rs b/rust/src/cancel.rs index aec4ac60..f53f7795 100644 --- a/rust/src/cancel.rs +++ b/rust/src/cancel.rs @@ -25,7 +25,9 @@ pub struct CancelToken { impl std::fmt::Debug for CancelToken { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("CancelToken").field("cancelled", &self.is_cancelled()).finish() + f.debug_struct("CancelToken") + .field("cancelled", &self.is_cancelled()) + .finish() } } @@ -47,7 +49,10 @@ impl CancelToken { return; } inner.cancelled = true; - (std::mem::take(&mut inner.wakers), std::mem::take(&mut inner.callbacks)) + ( + std::mem::take(&mut inner.wakers), + std::mem::take(&mut inner.callbacks), + ) }; for waker in wakers { waker.wake(); @@ -63,7 +68,9 @@ impl CancelToken { { let mut inner = self.inner.lock(); if !inner.cancelled { - inner.callbacks.push(callback.take().expect("callback present")); + inner + .callbacks + .push(callback.take().expect("callback present")); } } if let Some(callback) = callback { @@ -73,11 +80,14 @@ impl CancelToken { /// Completes once cancellation is requested. pub fn cancelled(&self) -> Cancelled { - Cancelled { token: self.clone() } + Cancelled { + token: self.clone(), + } } } /// Future returned by [`CancelToken::cancelled`]. +#[derive(Debug)] pub struct Cancelled { token: CancelToken, } diff --git a/rust/src/clock.rs b/rust/src/clock.rs index c089fc50..bbacccd7 100644 --- a/rust/src/clock.rs +++ b/rust/src/clock.rs @@ -47,7 +47,9 @@ mod system { let process = *PROCESS_ORIGIN.get_or_init(Instant::now); let now = Instant::now(); let phase = now.saturating_duration_since(process).as_nanos() % 1_000_000; - let origin = now.checked_sub(Duration::from_nanos(phase as u64)).unwrap_or(now); + let origin = now + .checked_sub(Duration::from_nanos(phase as u64)) + .unwrap_or(now); SystemClock { origin } } } diff --git a/rust/src/codec.rs b/rust/src/codec.rs index dcc77b38..76b5a4dc 100644 --- a/rust/src/codec.rs +++ b/rust/src/codec.rs @@ -21,11 +21,17 @@ pub struct Payload { impl Payload { pub fn text(text: impl Into) -> Self { - Payload { bytes: text.into().into_bytes(), binary: false } + Payload { + bytes: text.into().into_bytes(), + binary: false, + } } pub fn binary(bytes: impl Into>) -> Self { - Payload { bytes: bytes.into(), binary: true } + Payload { + bytes: bytes.into(), + binary: true, + } } pub fn len(&self) -> usize { diff --git a/rust/src/deadline.rs b/rust/src/deadline.rs new file mode 100644 index 00000000..445e1d7f --- /dev/null +++ b/rust/src/deadline.rs @@ -0,0 +1,59 @@ +//! Monotonic deadlines over controlled clocks. + +use std::time::Duration; + +use futures::future::{select, Either}; + +use crate::clock::Clock; +use crate::flight::Settled; + +/// Elapsed time since `start`, never negative. +pub(crate) fn since(clock: &dyn Clock, start: Duration) -> Duration { + clock.elapsed().saturating_sub(start) +} + +pub(crate) fn seconds_since(clock: &dyn Clock, start: Duration) -> f64 { + since(clock, start).as_secs_f64() +} + +/// Wait for `pending`, accepting only a result observed strictly before the +/// deadline `started + budget_ms`. Raw work keeps its resources after the +/// caller stops waiting. `None` budget waits without bound. +pub(crate) async fn await_deadline( + clock: &dyn Clock, + pending: &Settled, + started: Duration, + budget_ms: Option, + on_timeout: impl FnOnce() -> T, +) -> T { + let Some(budget_ms) = budget_ms else { + return pending.wait().await; + }; + let budget = Duration::from_millis(budget_ms); + loop { + if let Some(value) = pending.peek() { + if since(clock, started) < budget { + return value; + } + break; + } + let remaining = budget.saturating_sub(since(clock, started)); + let timer = clock.sleep(remaining); + match select(pending.wait(), timer).await { + Either::Left((value, _timer)) => { + if since(clock, started) < budget { + return value; + } + break; + } + Either::Right(((), _wait)) => { + // Timer precision and delivery do not define the semantic boundary. + if since(clock, started) < budget { + continue; + } + break; + } + } + } + on_timeout() +} diff --git a/rust/src/engine.rs b/rust/src/engine.rs new file mode 100644 index 00000000..2ce3d5e5 --- /dev/null +++ b/rust/src/engine.rs @@ -0,0 +1,687 @@ +//! The cache instance: construction, scopes, maintenance and call admission. + +use std::collections::{HashMap, HashSet}; +use std::future::Future; +use std::num::NonZeroUsize; +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use futures::future::BoxFuture; +use parking_lot::Mutex; + +use crate::clock::Clock; +use crate::error::{BoxError, ConfigError, Error, SharedError}; +use crate::flight::{start_pending, Flight, Settled, ValueResult}; +use crate::identity::Identity; +use crate::limits::{ + DEFAULT_LOCAL_CAPACITY, DEFAULT_REMOTE_READ_TIMEOUT_MS, DEFAULT_SHADOW_MAX_IN_FLIGHT, + MAX_DEADLINE_MS, MAX_SAFE_INTEGER, MAX_SUPPORTED_DURATION_MS, WATERMARK_USE_CASE, +}; +use crate::local::{LocalEntry, LocalStore, LruLocalStore, StoredValue}; +use crate::observe::{Event, Labels, Layer, LogEvent, Logger, Observer, OutcomeLabels}; +use crate::operation::{downcast_value, erase_load, ErasedOperation, Operation, RecoveryPredicate}; +use crate::policy::RuntimePolicy; +use crate::protocol::CompressionConfig; +use crate::remote::{InvalidateRequest, Remote}; +use crate::scope::{Owner, Scope}; +use crate::shadow::ShadowFlight; +use crate::spawn::Spawner; + +/// Resolves a sparse runtime policy overlay once per enabled call. +pub type PolicyProvider = Arc< + dyn Fn(Identity) -> BoxFuture<'static, Result, BoxError>> + Send + Sync, +>; + +static NEXT_CACHE_ID: AtomicU64 = AtomicU64::new(1); + +pub(crate) struct CoreState { + pub(crate) local: Option>, + pub(crate) flights: HashMap>, + pub(crate) shadows: HashMap>, + pub(crate) registered: HashSet, +} + +pub(crate) struct Core { + pub(crate) id: u64, + pub(crate) namespace: Arc, + pub(crate) remote: Option>, + pub(crate) observer: Option>, + pub(crate) logger: Arc, + pub(crate) provider: Option, + pub(crate) remote_read_timeout_ms: u64, + pub(crate) shadow_max_in_flight: usize, + pub(crate) should_recover: Option, + /// `None` disables new compressed writes; reads still decompress. + pub(crate) compression: Option, + pub(crate) clock: Arc, + pub(crate) spawner: Arc, + pub(crate) state: Mutex, +} + +impl Core { + /// Deliver a diagnostic. Observer failures never change results. + pub(crate) fn emit(&self, event: Event) { + if let Some(observer) = &self.observer { + let _ = catch_unwind(AssertUnwindSafe(|| observer.observe(&event))); + } + } + + pub(crate) fn log(&self, event: LogEvent) { + let _ = catch_unwind(AssertUnwindSafe(|| self.logger.log(&event))); + } + + pub(crate) fn shadow_hook_enabled(&self) -> bool { + match &self.observer { + Some(observer) => { + catch_unwind(AssertUnwindSafe(|| observer.observes_shadow_outcomes())) + .unwrap_or(false) + } + None => false, + } + } + + /// Read a live local entry, promoting it. `Ok(None)` is a miss. + pub(crate) fn local_get(&self, key: &str) -> Result, BoxError> { + // Read the clock before taking the lock: it is the only external code + // on this path, and a failure must not poison shared state. + let now_ms = self.clock.elapsed_ms(); + let mut state = self.state.lock(); + match state.local.as_mut() { + None => Ok(None), + Some(local) => catch_unwind(AssertUnwindSafe(|| local.get(key, now_ms))) + .unwrap_or_else(|payload| { + Err(crate::flight::panic_message(payload).to_string().into()) + }), + } + } + + pub(crate) fn local_put( + &self, + key: &str, + value: StoredValue, + ttl_ms: u64, + ) -> Result<(), BoxError> { + let now_ms = self.clock.elapsed_ms(); + let mut state = self.state.lock(); + match state.local.as_mut() { + None => Ok(()), + Some(local) => { + let entry = LocalEntry { + value, + inserted_ms: now_ms, + ttl_ms: ttl_ms.min(i64::MAX as u64) as i64, + }; + catch_unwind(AssertUnwindSafe(|| local.put(key.to_string(), entry))).unwrap_or_else( + |payload| Err(crate::flight::panic_message(payload).to_string().into()), + ) + } + } + } +} + +/// Exact process-scoped single-flight state of one instance. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct ProcessCoalescingState { + pub active_leaders: usize, + pub active_followers: usize, + pub oldest_leader_age_ms: Option, +} + +/// A point-in-time snapshot of cache-owned coalescing state. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct CoalescingState { + pub process: ProcessCoalescingState, +} + +/// Configures a [`DialCache`]. +pub struct DialCacheBuilder { + namespace: String, + remote: Option>, + observer: Option>, + logger: Option>, + provider: Option, + local_capacity: Option, + local_store: Option>, + remote_read_timeout_ms: Option, + shadow_max_in_flight: Option, + should_recover: Option, + compression: Option>, + clock: Option>, + spawner: Option>, +} + +impl std::fmt::Debug for DialCacheBuilder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DialCacheBuilder") + .field("namespace", &self.namespace) + .finish_non_exhaustive() + } +} + +impl DialCacheBuilder { + fn new() -> Self { + DialCacheBuilder { + namespace: "urn".to_string(), + remote: None, + observer: None, + logger: None, + provider: None, + local_capacity: None, + local_store: None, + remote_read_timeout_ms: None, + shadow_max_in_flight: None, + should_recover: None, + compression: None, + clock: None, + spawner: None, + } + } + + /// Logical namespace used in keys, invalidation identity, cohorts and + /// labels. Defaults to `"urn"`; may not contain `{` or `}`. + pub fn namespace(mut self, namespace: impl Into) -> Self { + self.namespace = namespace.into(); + self + } + + /// The remote layer adapter. Without one, only request and local layers serve. + pub fn remote(mut self, remote: Arc) -> Self { + self.remote = Some(remote); + self + } + + /// Receives every public diagnostic; also the shadow outcome hook. + pub fn observer(mut self, observer: Arc) -> Self { + self.observer = Some(observer); + self + } + + pub fn logger(mut self, logger: Arc) -> Self { + self.logger = Some(logger); + self + } + + /// Runtime policy overlay resolved once per enabled call. `Ok(None)` + /// inherits the operation policy; an error bypasses caching for that call. + pub fn policy_provider(mut self, provider: F) -> Self + where + F: Fn(Identity) -> Fut + Send + Sync + 'static, + Fut: Future, BoxError>> + Send + 'static, + { + self.provider = Some(Arc::new(move |identity| Box::pin(provider(identity)))); + self + } + + /// Maximum process-local entries across every use case. Zero disables + /// local storage while preserving coalescing. Defaults to 10,000. + pub fn local_capacity(mut self, capacity: usize) -> Self { + self.local_capacity = Some(capacity); + self + } + + /// Replace the default LRU store. Advanced: the store must preserve the + /// documented LRU, expiry and promotion rules. + pub fn local_store(mut self, store: Box) -> Self { + self.local_store = Some(store); + self + } + + /// Instance default remote read budget. Defaults to 50 ms. + pub fn remote_read_timeout_ms(mut self, ms: u64) -> Self { + self.remote_read_timeout_ms = Some(ms); + self + } + + /// Concurrent shadow jobs per instance; excess work is dropped. Defaults to 1. + pub fn shadow_max_in_flight(mut self, jobs: usize) -> Self { + self.shadow_max_in_flight = Some(jobs); + self + } + + /// Instance default classifier deciding whether a source failure may use + /// a retained stale value. Omission admits only the source deadline error. + pub fn should_recover( + mut self, + predicate: impl Fn(&Error) -> bool + Send + Sync + 'static, + ) -> Self { + self.should_recover = Some(Arc::new(predicate)); + self + } + + /// Write-side compression policy. Enabled by default at a 4,096-byte threshold. + pub fn compression(mut self, config: CompressionConfig) -> Self { + self.compression = Some(Some(config)); + self + } + + /// Store every payload uncompressed. Reads still accept compressed entries. + pub fn disable_compression(mut self) -> Self { + self.compression = Some(None); + self + } + + pub fn clock(mut self, clock: Arc) -> Self { + self.clock = Some(clock); + self + } + + pub fn spawner(mut self, spawner: Arc) -> Self { + self.spawner = Some(spawner); + self + } + + pub fn build(self) -> Result { + if self.namespace.contains(['{', '}']) { + return Err(ConfigError::invalid( + "DialCache namespace must not contain \"{\" or \"}\"", + )); + } + let compression = match self.compression { + None => Some(CompressionConfig::default()), + Some(None) => None, + Some(Some(config)) => { + config + .validate() + .map_err(|e| ConfigError::invalid(e.to_string()))?; + Some(config) + } + }; + let remote_read_timeout_ms = self + .remote_read_timeout_ms + .unwrap_or(DEFAULT_REMOTE_READ_TIMEOUT_MS); + if !(1..=MAX_DEADLINE_MS).contains(&remote_read_timeout_ms) { + return Err(ConfigError::invalid(format!( + "DialCache remote read timeout must be a positive integer no greater than {MAX_DEADLINE_MS} ms" + ))); + } + let shadow_max_in_flight = self + .shadow_max_in_flight + .unwrap_or(DEFAULT_SHADOW_MAX_IN_FLIGHT); + if shadow_max_in_flight == 0 { + return Err(ConfigError::invalid( + "DialCache shadow_max_in_flight must be positive", + )); + } + let local: Option> = match self.local_store { + Some(store) => Some(store), + None => { + let capacity = self.local_capacity.unwrap_or(DEFAULT_LOCAL_CAPACITY); + if capacity as u64 > MAX_SAFE_INTEGER { + return Err(ConfigError::invalid( + "DialCache local capacity must be a safe integer", + )); + } + NonZeroUsize::new(capacity) + .map(|capacity| Box::new(LruLocalStore::new(capacity)) as Box) + } + }; + let clock = match self.clock { + Some(clock) => clock, + None => default_clock()?, + }; + let spawner = match self.spawner { + Some(spawner) => spawner, + None => default_spawner()?, + }; + let core = Core { + id: NEXT_CACHE_ID.fetch_add(1, Ordering::Relaxed), + namespace: Arc::from(self.namespace.as_str()), + remote: self.remote, + observer: self.observer, + logger: self + .logger + .unwrap_or_else(|| Arc::new(crate::observe::LogFacadeLogger)), + provider: self.provider, + remote_read_timeout_ms, + shadow_max_in_flight, + should_recover: self.should_recover, + compression, + clock, + spawner, + state: Mutex::new(CoreState { + local, + flights: HashMap::new(), + shadows: HashMap::new(), + registered: HashSet::new(), + }), + }; + Ok(DialCache { + core: Arc::new(core), + }) + } +} + +#[cfg(feature = "tokio")] +fn default_clock() -> Result, ConfigError> { + Ok(Arc::new(crate::clock::SystemClock::new())) +} + +#[cfg(not(feature = "tokio"))] +fn default_clock() -> Result, ConfigError> { + Err(ConfigError::invalid("DialCache needs a clock: enable the tokio feature or supply one with DialCacheBuilder::clock")) +} + +#[cfg(feature = "tokio")] +fn default_spawner() -> Result, ConfigError> { + Ok(Arc::new(crate::spawn::TokioSpawner)) +} + +#[cfg(not(feature = "tokio"))] +fn default_spawner() -> Result, ConfigError> { + Err(ConfigError::invalid("DialCache needs a spawner: enable the tokio feature or supply one with DialCacheBuilder::spawner")) +} + +/// A cache instance: request, process-local and remote layers behind explicit enablement. +/// +/// Instances are cheap to clone and share one state. +#[derive(Clone)] +pub struct DialCache { + pub(crate) core: Arc, +} + +impl std::fmt::Debug for DialCache { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DialCache") + .field("namespace", &self.core.namespace) + .finish_non_exhaustive() + } +} + +impl DialCache { + pub fn builder() -> DialCacheBuilder { + DialCacheBuilder::new() + } + + /// The instance namespace. + pub fn namespace(&self) -> &str { + &self.core.namespace + } + + /// Open the outermost enabled scope for `f`. The scope closes when the + /// returned future completes; retained clones then pass through. + pub async fn enable(&self, f: F) -> R + where + F: FnOnce(Scope) -> Fut, + Fut: Future, + { + let owner = Owner::new(); + let scope = Scope { + cache_id: self.core.id, + owner: Some(owner.clone()), + enabled: true, + }; + let result = f(scope).await; + owner.close(); + result + } + + /// Enable caching inside `parent`, reusing its live outer request memo. + /// When `parent` has no live outer scope, this opens a new one. + pub async fn enable_in(&self, parent: &Scope, f: F) -> R + where + F: FnOnce(Scope) -> Fut, + Fut: Future, + { + let live = if parent.cache_id == self.core.id { + parent.live_owner() + } else { + None + }; + match live { + Some(owner) => { + let scope = Scope { + cache_id: self.core.id, + owner: Some(owner), + enabled: true, + }; + f(scope).await + } + None => self.enable(f).await, + } + } + + /// Disable caching inside `parent` while preserving its outer request memo + /// for nested re-enablement. + pub async fn disable_in(&self, parent: &Scope, f: F) -> R + where + F: FnOnce(Scope) -> Fut, + Fut: Future, + { + let owner = if parent.cache_id == self.core.id { + parent.owner.clone() + } else { + None + }; + let scope = Scope { + cache_id: self.core.id, + owner, + enabled: false, + }; + f(scope).await + } + + /// Whether calls made with `scope` on this instance use caching. + pub fn is_enabled(&self, scope: &Scope) -> bool { + scope.cache_id == self.core.id && scope.is_enabled() + } + + /// Process-scoped single-flight state of this instance. + pub fn coalescing_state(&self) -> CoalescingState { + let now = self.core.clock.elapsed(); + let state = self.core.state.lock(); + let mut process = ProcessCoalescingState { + active_leaders: state.flights.len(), + ..Default::default() + }; + for flight in state.flights.values() { + process.active_followers += flight.followers(); + let age = now + .saturating_sub(flight.started) + .as_millis() + .min(u64::MAX as u128) as u64; + if process + .oldest_leader_age_ms + .is_none_or(|oldest| age > oldest) + { + process.oldest_leader_age_ms = Some(age); + } + } + CoalescingState { process } + } + + /// Advance the invalidation watermark of every tracked variant of one + /// entity, in this instance's namespace. Call it after the source + /// mutation commits. Requires a remote adapter; failures are returned. + pub async fn invalidate( + &self, + key_type: &str, + id: &str, + future_buffer_ms: u64, + ) -> Result<(), Error> { + let identity = Identity::new(key_type, id, WATERMARK_USE_CASE) + .tracked(true) + .namespace(self.core.namespace.to_string()); + self.invalidate_identity(identity, future_buffer_ms).await + } + + async fn invalidate_identity( + &self, + mut identity: Identity, + future_buffer_ms: u64, + ) -> Result<(), Error> { + let core = &self.core; + if future_buffer_ms > MAX_SUPPORTED_DURATION_MS { + return Err(ConfigError::invalid( + "DialCache invalidation future buffer must be no greater than 365 days", + ) + .into()); + } + if identity.namespace.is_empty() { + identity.namespace = core.namespace.to_string(); + } + identity.tracked = true; + let namespace: Arc = Arc::from(identity.namespace.as_str()); + let key_type: Arc = Arc::from(identity.key_type.as_str()); + core.emit(Event::Invalidation { + namespace: namespace.clone(), + key_type: key_type.clone(), + layer: Layer::Remote, + }); + let result: Result<(), Error> = async { + let remote = core.remote.clone().ok_or(Error::MissingRemote)?; + let keys = identity + .keys() + .map_err(|e| Error::Config(ConfigError::invalid(e.to_string())))?; + let watermark_key = keys.watermark.ok_or_else(|| { + Error::Config(ConfigError::invalid( + "tracked identity has no watermark key", + )) + })?; + let now = core.clock.wall_ms(); + if now < 0 || now as u64 > MAX_SAFE_INTEGER - future_buffer_ms { + return Err(ConfigError::invalid( + "DialCache invalidation timestamp is outside the safe integer domain", + ) + .into()); + } + let request = InvalidateRequest { + watermark_key, + invalidated_at_ms: now as u64, + future_buffer_ms, + }; + let pending: Settled> = start_pending( + core.spawner.as_ref(), + async move { + remote + .invalidate(request) + .await + .map_err(|e| Error::Remote(Arc::from(e))) + }, + |message| Err(Error::Panic(message)), + ); + pending.wait().await + } + .await; + if let Err(error) = &result { + core.log(LogEvent::InvalidationFailed(error.to_string().into())); + core.emit(Event::Error { + labels: Labels { + namespace, + use_case: Arc::from(WATERMARK_USE_CASE), + key_type, + layer: Layer::Remote, + }, + error: crate::observe::ErrorKind::Invalidation, + in_fallback: false, + }); + } + result + } + + /// Execute one inline cached call. + /// + /// `load` is the source of truth; it may be invoked again later by + /// served-hit shadow validation, so it must be reusable. The returned + /// value is shared by reference: treat it as immutable. + pub async fn get_or_load( + &self, + scope: &Scope, + operation: Operation, + load: F, + ) -> Result, Error> + where + T: Send + Sync + 'static, + F: Fn(Scope) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { + let parts = operation.erase(); + let erased = ErasedOperation { + identity: parts.identity, + identity_provider: None, + policy: parts.policy, + budget: parts.budget, + codec: parts.codec, + compare: parts.compare, + should_recover: parts.should_recover, + preview: parts.preview, + load: erase_load(load), + }; + let value = self.execute(scope, erased).await?; + downcast_value::(value) + } + + /// Validate static operation configuration, then run the execution as + /// detached work whose result every caller awaits. Dropping the returned + /// future does not cancel the execution: sources and publications keep + /// their contracts, as in the reference implementations. + pub(crate) async fn execute( + &self, + scope: &Scope, + mut operation: ErasedOperation, + ) -> ValueResult { + validate_operation(&operation)?; + if operation.identity.namespace.is_empty() { + operation.identity.namespace = self.core.namespace.to_string(); + } + let core = self.core.clone(); + let scope = if scope.cache_id == core.id { + scope.clone() + } else { + Scope::outside() + }; + let pending: Settled = start_pending( + core.spawner.clone().as_ref(), + crate::execution::run(core, scope, Arc::new(operation)), + |message| Err(Error::Panic(message)), + ); + pending.wait().await + } + + pub(crate) fn register_use_case(&self, use_case: &str) -> Result<(), ConfigError> { + if use_case == WATERMARK_USE_CASE { + return Err(ConfigError::ReservedUseCase(use_case.to_string())); + } + let mut state = self.core.state.lock(); + if !state.registered.insert(use_case.to_string()) { + return Err(ConfigError::UseCaseAlreadyRegistered(use_case.to_string())); + } + Ok(()) + } +} + +pub(crate) fn validate_operation(operation: &ErasedOperation) -> Result<(), Error> { + operation + .policy + .validate() + .map_err(|e| ConfigError::invalid(e.to_string()))?; + if let crate::operation::SourceBudget::Millis(ms) = operation.budget { + if !(1..=MAX_DEADLINE_MS).contains(&ms) { + return Err(ConfigError::invalid(format!( + "DialCache source budget must be a positive integer no greater than {MAX_DEADLINE_MS} ms" + )) + .into()); + } + } + if operation.identity.use_case == WATERMARK_USE_CASE { + return Err(ConfigError::ReservedUseCase(operation.identity.use_case.clone()).into()); + } + Ok(()) +} + +pub(crate) fn outcome_labels(identity: &Identity) -> OutcomeLabels { + OutcomeLabels { + namespace: Arc::from(identity.namespace.as_str()), + use_case: Arc::from(identity.use_case.as_str()), + key_type: Arc::from(identity.key_type.as_str()), + } +} + +#[allow(dead_code)] +pub(crate) fn shared_error(error: BoxError) -> SharedError { + Arc::from(error) +} + +#[allow(dead_code)] +pub(crate) struct Unused(AtomicBool, Duration); diff --git a/rust/src/error.rs b/rust/src/error.rs index e4762018..9399bc31 100644 --- a/rust/src/error.rs +++ b/rust/src/error.rs @@ -45,10 +45,6 @@ impl Error { pub fn is_fallback_timeout(&self) -> bool { matches!(self, Error::FallbackTimeout(_)) } - - pub(crate) fn from_box(error: BoxError) -> Error { - Error::Source(Arc::from(error)) - } } impl fmt::Display for Error { @@ -58,7 +54,9 @@ impl fmt::Display for Error { Error::FallbackTimeout(timeout) => timeout.fmt(f), Error::Panic(message) => write!(f, "DialCache callback panicked: {message}"), Error::Config(error) => error.fmt(f), - Error::MissingRemote => f.write_str("DialCache invalidation requires a configured remote adapter"), + Error::MissingRemote => { + f.write_str("DialCache invalidation requires a configured remote adapter") + } Error::Remote(error) => write!(f, "DialCache remote maintenance failed: {error}"), } } diff --git a/rust/src/execution.rs b/rust/src/execution.rs new file mode 100644 index 00000000..f95f48aa --- /dev/null +++ b/rust/src/execution.rs @@ -0,0 +1,994 @@ +//! One enabled call: admission, policy capture, traversal, source, publication +//! and stale recovery. Ported transition by transition from the reference +//! implementations; the Quint models define the behavior. + +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use futures::future::BoxFuture; +use futures::FutureExt; + +use crate::cancel::CancelToken; +use crate::deadline::{await_deadline, seconds_since}; +use crate::engine::Core; +use crate::error::{BoxError, Error, FallbackTimeout, RemoteReadTimeout, SharedError}; +use crate::flight::{panic_message, start_pending, Flight, Settled, ValueResult}; +use crate::identity::{Identity, Keys}; +use crate::limits::{MAX_DECOMPRESSED_BYTES, MAX_SAFE_INTEGER, MAX_TRACKED_VALUE_TTL_MS}; +use crate::local::StoredValue; +use crate::observe::{ + CoalescingScope, CompressionOperation, DisabledReason, ErrorKind, Event, Labels, Layer, + LogEvent, MissReason, OutcomeLabels, RecoveryOutcome, SerializationOperation, +}; +use crate::operation::ErasedOperation; +use crate::policy::{resolve_policy, PolicyDefaults, ResolvedLayer, ResolvedPolicy}; +use crate::protocol::{ + compress_payload, decompress_payload, escape_raw_payload, normalize_read_result, +}; +use crate::remote::{Frame, ReadContext, ReadRequest, ReadResult, WriteRequest}; +use crate::scope::{Owner, Scope}; + +/// The raw outcome of one adapter read. +pub(crate) type RawRead = Result; + +#[derive(Debug, thiserror::Error)] +#[error("DialCache callback panicked: {0}")] +pub(crate) struct PanicError(pub Arc); + +/// One admitted enabled call. +pub(crate) struct Execution { + pub(crate) core: Arc, + pub(crate) scope: Scope, + pub(crate) op: Arc, + pub(crate) identity: Identity, + pub(crate) keys: Keys, + pub(crate) policy: ResolvedPolicy, + pub(crate) labels: OutcomeLabels, + pub(crate) timed_out: Arc, +} + +/// What the serving remote read produced. +pub(crate) enum RemoteValue { + Hit { + value: StoredValue, + frame: Frame, + }, + /// A valid stale frame retained only as a recovery candidate. + Retained { + frame: Frame, + }, + Miss { + fence: Option, + }, + DecodeError, + Error, +} + +/// Run one call from admission to result. +pub(crate) async fn run(core: Arc, scope: Scope, op: Arc) -> ValueResult { + let mut identity = op.identity.clone(); + let noop_labels = crate::engine::outcome_labels(&identity); + let noop = |reason: DisabledReason| Event::Disabled { + labels: layer_labels(&noop_labels, Layer::Noop), + reason, + }; + if !core.id_enabled(&scope) { + core.emit(noop(DisabledReason::Context)); + return call_load(&op, scope).await; + } + if let Some(provider) = &op.identity_provider { + let computed = catch_unwind(AssertUnwindSafe(|| provider())) + .unwrap_or_else(|payload| Err(Box::new(PanicError(panic_message(payload))) as BoxError)) + .and_then(|identity| { + if identity.use_case == crate::limits::WATERMARK_USE_CASE { + Err("reserved use case: watermark".into()) + } else { + Ok(identity) + } + }); + match computed { + Ok(mut computed) => { + if computed.namespace.is_empty() { + computed.namespace = core.namespace.to_string(); + } + identity = computed; + } + Err(error) => { + core.log(LogEvent::KeyConstructionFailed(error)); + core.emit(Event::Error { + labels: layer_labels(&noop_labels, Layer::Noop), + error: ErrorKind::KeyConstruction, + in_fallback: false, + }); + return uncached_source(&core, &op, scope, &noop_labels).await; + } + } + } + let labels = crate::engine::outcome_labels(&identity); + let keys = match identity.keys() { + Ok(keys) => keys, + Err(error) => { + core.log(LogEvent::KeyConstructionFailed(Box::new(error))); + core.emit(Event::Error { + labels: layer_labels(&labels, Layer::Noop), + error: ErrorKind::KeyConstruction, + in_fallback: false, + }); + return uncached_source(&core, &op, scope, &labels).await; + } + }; + let overlay: Result, BoxError> = match &core.provider { + Some(provider) => { + let pending = start_pending( + core.spawner.as_ref(), + { + let provider = provider.clone(); + let identity = identity.clone(); + async move { + provider(identity) + .await + .map_err(|e| Arc::from(e) as SharedError) + } + }, + |message| Err(Arc::new(PanicError(message)) as SharedError), + ); + pending + .wait() + .await + .map_err(|e| Box::new(SharedErrorWrapper(e)) as BoxError) + } + None => Ok(None), + }; + let resolved = overlay.and_then(|overlay| { + resolve_policy( + &op.policy, + overlay.as_ref(), + &keys.logical, + PolicyDefaults { + remote_read_timeout_ms: core.remote_read_timeout_ms, + }, + ) + .map_err(|e| Box::new(e) as BoxError) + }); + let policy = match resolved { + Ok(policy) => policy, + Err(error) => { + core.log(LogEvent::PolicyResolutionFailed(error)); + core.emit(Event::Error { + labels: layer_labels(&labels, Layer::Noop), + error: ErrorKind::ConfigResolution, + in_fallback: false, + }); + core.emit(Event::Disabled { + labels: layer_labels(&labels, Layer::Noop), + reason: DisabledReason::ConfigError, + }); + return uncached_source(&core, &op, scope, &labels).await; + } + }; + if !core.id_enabled(&scope) { + core.emit(Event::Disabled { + labels: layer_labels(&labels, Layer::Noop), + reason: DisabledReason::Context, + }); + return uncached_source(&core, &op, scope, &labels).await; + } + let execution = Arc::new(Execution { + core: core.clone(), + scope: scope.clone(), + op, + identity, + keys, + policy, + labels, + timed_out: Arc::new(AtomicBool::new(false)), + }); + if !execution.policy.request_local { + return execution.shared(Layer::Local).await; + } + let Some(owner) = scope.owner.clone() else { + return execution.shared(Layer::Local).await; + }; + let request_run = { + let x = execution.clone(); + let owner = owner.clone(); + move || -> BoxFuture<'static, ValueResult> { + Box::pin(async move { x.through_request(owner).await }) + } + }; + if !execution.policy.coalesce { + return request_run().await; + } + execution + .single_flight(Some(owner), CoalescingScope::RequestLocal, request_run) + .await +} + +#[derive(Debug)] +struct SharedErrorWrapper(SharedError); + +impl std::fmt::Display for SharedErrorWrapper { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +impl std::error::Error for SharedErrorWrapper {} + +impl Core { + pub(crate) fn id_enabled(&self, scope: &Scope) -> bool { + scope.cache_id == self.id && scope.is_enabled() + } +} + +pub(crate) fn layer_labels(labels: &OutcomeLabels, layer: Layer) -> Labels { + Labels { + namespace: labels.namespace.clone(), + use_case: labels.use_case.clone(), + key_type: labels.key_type.clone(), + layer, + } +} + +/// Invoke the source directly with panic isolation; errors keep their identity. +async fn call_load(op: &ErasedOperation, scope: Scope) -> ValueResult { + let future = (op.load)(scope); + match AssertUnwindSafe(future).catch_unwind().await { + Ok(Ok(value)) => Ok(value), + Ok(Err(error)) => Err(Error::Source(Arc::from(error))), + Err(payload) => Err(Error::Panic(panic_message(payload))), + } +} + +/// The enabled but uncached path: the source runs with its enabled deadline. +async fn uncached_source( + core: &Arc, + op: &Arc, + scope: Scope, + labels: &OutcomeLabels, +) -> ValueResult { + source_with_budget(core, op, scope, labels, Layer::Noop, None).await +} + +/// Run the source under the operation budget, reporting the fallback trail. +async fn source_with_budget( + core: &Arc, + op: &Arc, + scope: Scope, + labels: &OutcomeLabels, + layer: Layer, + timed_out: Option<&AtomicBool>, +) -> ValueResult { + let clock = core.clock.clone(); + let start = clock.elapsed(); + let budget = op.budget.millis(); + let pending: Settled = start_pending( + core.spawner.as_ref(), + { + let op = op.clone(); + async move { call_load(&op, scope).await } + }, + |message| Err(Error::Panic(message)), + ); + let use_case = labels.use_case.to_string(); + let result = await_deadline(clock.as_ref(), &pending, start, budget, || { + if let Some(flag) = timed_out { + flag.store(true, Ordering::SeqCst); + } + Err(Error::FallbackTimeout(Arc::new(FallbackTimeout { + use_case, + timeout_ms: budget.unwrap_or(0), + }))) + }) + .await; + if result.is_err() { + core.emit(Event::Error { + labels: layer_labels(labels, layer), + error: ErrorKind::Fallback, + in_fallback: true, + }); + } + core.emit(Event::Fallback { + labels: layer_labels(labels, layer), + seconds: seconds_since(clock.as_ref(), start), + }); + result +} + +impl Execution { + pub(crate) fn labels(&self, layer: Layer) -> Labels { + layer_labels(&self.labels, layer) + } + + pub(crate) fn emit(&self, event: Event) { + self.core.emit(event); + } + + pub(crate) fn error_event(&self, layer: Layer, error: ErrorKind, in_fallback: bool) { + self.emit(Event::Error { + labels: self.labels(layer), + error, + in_fallback, + }); + } + + pub(crate) fn elapsed(&self) -> Duration { + self.core.clock.elapsed() + } + + pub(crate) fn seconds_since(&self, start: Duration) -> f64 { + seconds_since(self.core.clock.as_ref(), start) + } + + pub(crate) fn wall_ms(&self) -> i64 { + self.core.clock.wall_ms() + } + + /// The source under the operation budget, attributed to `layer`. + pub(crate) async fn source(&self, layer: Layer) -> ValueResult { + source_with_budget( + &self.core, + &self.op, + self.scope.clone(), + &self.labels, + layer, + Some(&self.timed_out), + ) + .await + } + + async fn through_request(self: Arc, owner: Arc) -> ValueResult { + let start = self.elapsed(); + let (memo, live) = { + let state = owner.state.lock(); + (state.memo.get(&self.keys.logical).cloned(), state.live) + }; + self.emit(Event::Request { + labels: self.labels(Layer::RequestLocal), + }); + self.emit(Event::Get { + labels: self.labels(Layer::RequestLocal), + seconds: self.seconds_since(start), + }); + if let (true, Some(value)) = (live, memo) { + return Ok(value); + } + self.emit(Event::Miss { + labels: self.labels(Layer::RequestLocal), + reason: MissReason::ValueAbsent, + }); + let result = self.clone().shared(Layer::RequestLocal).await; + if let Ok(value) = &result { + let mut state = owner.state.lock(); + if state.live { + state.memo.insert(self.keys.logical.clone(), value.clone()); + } + } + result + } + + /// Join an existing execution for this key or lead a new one. + pub(crate) async fn single_flight( + self: &Arc, + owner: Option>, + label: CoalescingScope, + run: F, + ) -> ValueResult + where + F: FnOnce() -> BoxFuture<'static, ValueResult>, + { + // Read the clock before taking any lock: it is external code. + let started = self.elapsed(); + let key = self.keys.logical.clone(); + let flight = { + let joined = match &owner { + Some(owner) => { + let mut state = owner.state.lock(); + if !state.live { + None + } else if let Some(existing) = state.flights.get(&key) { + existing.join(); + Some(Ok(existing.clone())) + } else { + let flight = Flight::new(started); + state.flights.insert(key.clone(), flight.clone()); + Some(Err(flight)) + } + } + None => { + let mut state = self.core.state.lock(); + if let Some(existing) = state.flights.get(&key) { + existing.join(); + Some(Ok(existing.clone())) + } else { + let flight = Flight::new(started); + state.flights.insert(key.clone(), flight.clone()); + Some(Err(flight)) + } + } + }; + match joined { + None => return run().await, + Some(Ok(existing)) => { + self.emit(Event::Coalesced { + labels: self.labels.clone(), + scope: label, + }); + return existing.result.wait().await; + } + Some(Err(flight)) => flight, + } + }; + let result = match AssertUnwindSafe(run()).catch_unwind().await { + Ok(result) => result, + Err(payload) => Err(Error::Panic(panic_message(payload))), + }; + { + match &owner { + Some(owner) => { + let mut state = owner.state.lock(); + if state + .flights + .get(&key) + .is_some_and(|f| Arc::ptr_eq(f, &flight)) + { + state.flights.remove(&key); + } + } + None => { + let mut state = self.core.state.lock(); + if state + .flights + .get(&key) + .is_some_and(|f| Arc::ptr_eq(f, &flight)) + { + state.flights.remove(&key); + } + } + } + flight.result.settle(result.clone()); + } + result + } + + fn layer_event(&self, layer: Layer, resolved: &ResolvedLayer) { + if !resolved.enabled { + let reason = resolved.reason.unwrap_or(DisabledReason::PolicyDisabled); + self.emit(Event::Disabled { + labels: self.labels(layer), + reason, + }); + if matches!( + reason, + DisabledReason::InvalidTtl | DisabledReason::InvalidRamp + ) { + self.error_event(layer, ErrorKind::ConfigResolution, false); + } + } + } + + /// Traverse the shared layers below the request layer. + pub(crate) async fn shared(self: Arc, fallback_layer: Layer) -> ValueResult { + let p = self.policy.clone(); + self.layer_event(Layer::Local, &p.local); + let run = { + let x = self.clone(); + move || -> BoxFuture<'static, ValueResult> { + Box::pin(async move { x.shared_run(fallback_layer).await }) + } + }; + if p.local.enabled { + if p.coalesce { + return self + .single_flight(None, CoalescingScope::Process, run) + .await; + } + return run().await; + } + // Remote admission is decided before joining; traversal belongs to the leader. + if p.remote.enabled && self.core.remote.is_some() && p.coalesce { + return self + .single_flight(None, CoalescingScope::Process, run) + .await; + } + run().await + } + + async fn shared_run(self: Arc, mut fallback_layer: Layer) -> ValueResult { + let p = self.policy.clone(); + let mut local_miss = false; + if p.local.enabled { + let start = self.elapsed(); + match self.core.local_get(&self.keys.logical) { + Err(error) => { + self.core.log(LogEvent::LocalReadFailed(error)); + self.error_event(Layer::Local, ErrorKind::CacheRead, false); + self.emit(Event::Disabled { + labels: self.labels(Layer::Local), + reason: DisabledReason::ConfigError, + }); + } + Ok(found) => { + self.emit(Event::Request { + labels: self.labels(Layer::Local), + }); + self.emit(Event::Get { + labels: self.labels(Layer::Local), + seconds: self.seconds_since(start), + }); + if let Some(value) = found { + return Ok(value); + } + local_miss = true; + self.emit(Event::Miss { + labels: self.labels(Layer::Local), + reason: MissReason::ValueAbsent, + }); + } + } + fallback_layer = Layer::Local; + } + if self.core.remote.is_none() { + let result = self.source(fallback_layer).await; + if let Ok(value) = &result { + if local_miss { + self.put_local(value.clone()); + } + } + return result; + } + if p.stale_on_error_config_error { + self.error_event(Layer::Remote, ErrorKind::ConfigResolution, false); + } + self.layer_event(Layer::Remote, &p.remote); + if !p.remote.enabled { + if p.remote.reason == Some(DisabledReason::RampedDown) { + return self.dark_source(fallback_layer, local_miss).await; + } + let result = self.source(fallback_layer).await; + if let Ok(value) = &result { + if local_miss { + self.put_local(value.clone()); + } + } + return result; + } + let remote = self.read_serving().await; + if let RemoteValue::Hit { value, frame } = remote { + if local_miss { + self.put_local(value.clone()); + } + self.schedule_shadow(Some(frame), None, Duration::ZERO); + return Ok(value); + } + let result = self.source(Layer::Remote).await; + match result { + Err(error) => { + let recoverable = !matches!(remote, RemoteValue::Error | RemoteValue::DecodeError); + if recoverable && p.stale_on_error_max_age_ms > 0 && self.can_recover(&error) { + let frame = match &remote { + RemoteValue::Retained { frame } => Some(frame), + _ => None, + }; + if let Some(value) = self.recover(frame).await { + return Ok(value); + } + } + Err(error) + } + Ok(value) => { + if !matches!(remote, RemoteValue::Error) { + let fence = match &remote { + RemoteValue::Miss { fence } => *fence, + _ => None, + }; + if let Err(error) = self.put_remote(&value, fence, Layer::Remote, None).await { + self.core.log(LogEvent::RemoteWriteFailed(error)); + } + } + if local_miss && !self.identity.tracked { + self.put_local(value.clone()); + } + Ok(value) + } + } + } + + pub(crate) fn put_local(&self, value: StoredValue) { + if let Err(error) = self + .core + .local_put(&self.keys.logical, value, self.policy.local.ttl_ms) + { + self.core.log(LogEvent::LocalWriteFailed(error)); + self.error_event(Layer::Local, ErrorKind::CacheWrite, false); + } + } + + /// Start one adapter read under the read budget. Returns the bounded + /// result and the raw work, which keeps running after a deadline. + pub(crate) fn raw_read(&self) -> (BoxFuture<'static, RawRead>, Settled) { + let remote = self + .core + .remote + .clone() + .expect("remote layer requires an adapter"); + let cancel = CancelToken::new(); + let timeout_ms = self.policy.remote_read_timeout_ms; + let context = ReadContext { + timeout_ms, + cancel: cancel.clone(), + }; + let request = ReadRequest { + value_key: self.keys.value.clone(), + watermark_key: if self.identity.tracked { + self.keys.watermark.clone() + } else { + None + }, + }; + let clock = self.core.clock.clone(); + let start = clock.elapsed(); + let raw: Settled = start_pending( + self.core.spawner.as_ref(), + async move { + remote + .read(request, context) + .await + .map_err(|e| Arc::from(e) as SharedError) + }, + |message| Err(Arc::new(PanicError(message)) as SharedError), + ); + let tracked = self.identity.tracked; + let use_case = self.labels.use_case.to_string(); + let waited = raw.clone(); + let bounded = Box::pin(async move { + let result = await_deadline(clock.as_ref(), &waited, start, Some(timeout_ms), || { + cancel.cancel(); + Err(Arc::new(RemoteReadTimeout { + use_case, + timeout_ms, + }) as SharedError) + }) + .await; + result.map(|read| normalize_read_result(read, tracked)) + }); + (bounded, raw) + } + + /// Age of a frame against the wall clock; `false` for invalid or future stamps. + pub(crate) fn frame_age(&self, frame: &Frame, layer: Layer) -> (i64, bool) { + if frame.created_at_ms > MAX_SAFE_INTEGER { + return (0, false); + } + let age = self.wall_ms().saturating_sub(frame.created_at_ms as i64); + if age < 0 { + self.emit(Event::FutureTimestampOffset { + labels: self.labels(layer), + seconds: (-age) as f64 / 1000.0, + }); + return (age, false); + } + (age, true) + } + + async fn read_serving(&self) -> RemoteValue { + let start = self.elapsed(); + self.emit(Event::Request { + labels: self.labels(Layer::Remote), + }); + let (bounded, _raw) = self.raw_read(); + let result = bounded.await; + let value = match result { + Err(error) => { + let kind = if error.downcast_ref::().is_some() { + ErrorKind::CacheReadTimeout + } else { + ErrorKind::CacheRead + }; + self.core + .log(LogEvent::RemoteReadFailed(Box::new(SharedErrorWrapper( + error, + )))); + self.error_event(Layer::Remote, kind, false); + RemoteValue::Error + } + Ok(ReadResult::Miss { + reason, + observed_watermark_ms, + }) => { + self.emit(Event::Miss { + labels: self.labels(Layer::Remote), + reason, + }); + RemoteValue::Miss { + fence: observed_watermark_ms, + } + } + Ok(ReadResult::Hit(frame)) => { + let (age, valid) = self.frame_age(&frame, Layer::Remote); + if !valid { + self.emit(Event::Miss { + labels: self.labels(Layer::Remote), + reason: MissReason::Unclassified, + }); + RemoteValue::Miss { fence: None } + } else { + let max_age = if self.policy.stale_on_error_max_age_ms > 0 { + self.policy.stale_on_error_max_age_ms + } else { + self.policy.remote.ttl_ms + }; + if age as u64 >= max_age { + self.emit(Event::Miss { + labels: self.labels(Layer::Remote), + reason: MissReason::Expired, + }); + RemoteValue::Miss { fence: None } + } else if age as u64 >= self.policy.remote.ttl_ms { + self.emit(Event::Miss { + labels: self.labels(Layer::Remote), + reason: MissReason::Expired, + }); + RemoteValue::Retained { frame } + } else { + match self.decode(&frame, Layer::Remote).await { + Ok(value) => RemoteValue::Hit { value, frame }, + Err(_) => { + self.emit(Event::Miss { + labels: self.labels(Layer::Remote), + reason: MissReason::Unclassified, + }); + RemoteValue::DecodeError + } + } + } + } + } + }; + self.emit(Event::Get { + labels: self.labels(Layer::Remote), + seconds: self.seconds_since(start), + }); + value + } + + /// Decode a frame's payload: envelope, then codec, with diagnostics. + pub(crate) async fn decode( + &self, + frame: &Frame, + layer: Layer, + ) -> Result { + let decompress_started = self.elapsed(); + let expanded = decompress_payload(frame.payload.clone(), MAX_DECOMPRESSED_BYTES); + if let Some(outcome) = expanded.outcome { + self.emit(Event::Compression { + labels: self.labels(layer), + outcome, + }); + self.emit(Event::CompressionDuration { + labels: self.labels(layer), + operation: CompressionOperation::Decompress, + seconds: self.seconds_since(decompress_started), + }); + } + let start = self.elapsed(); + let codec = self.op.codec.clone(); + let decoded = match AssertUnwindSafe(codec.decode(expanded.payload)) + .catch_unwind() + .await + { + Ok(result) => result, + Err(payload) => Err(Box::new(PanicError(panic_message(payload))) as BoxError), + }; + if decoded.is_err() { + self.error_event(layer, ErrorKind::SerializationLoad, false); + } + self.emit(Event::Serialization { + labels: self.labels(layer), + operation: SerializationOperation::Load, + seconds: self.seconds_since(start), + }); + decoded + } + + fn write_timestamp(&self, layer: Layer) -> Result { + let stamp = self.wall_ms(); + if stamp < 0 || stamp as u64 > MAX_SAFE_INTEGER { + self.error_event(layer, ErrorKind::CacheWrite, false); + return Err("DialCache write timestamp is outside the safe integer domain".into()); + } + Ok(stamp as u64) + } + + /// Prepare and dispatch one remote write. `Ok(false)` means the write was + /// fenced or refused by `allowed`; `Ok(true)` means it was dispatched. + pub(crate) async fn put_remote( + &self, + value: &StoredValue, + fence: Option, + layer: Layer, + allowed: Option<&(dyn Fn() -> bool + Send + Sync)>, + ) -> Result { + let fence = if self.identity.tracked { fence } else { None }; + if let Some(fence) = fence { + let stamp = self.write_timestamp(layer)?; + if stamp <= fence { + return Ok(false); + } + } + let start = self.elapsed(); + let codec = self.op.codec.clone(); + let encoded = match AssertUnwindSafe(codec.encode(value)).catch_unwind().await { + Ok(result) => result, + Err(payload) => Err(Box::new(PanicError(panic_message(payload))) as BoxError), + }; + if encoded.is_err() { + self.error_event(layer, ErrorKind::SerializationDump, false); + } + self.emit(Event::Serialization { + labels: self.labels(layer), + operation: SerializationOperation::Dump, + seconds: self.seconds_since(start), + }); + let mut payload = encoded?; + self.emit(Event::Size { + labels: self.labels(layer), + bytes: payload.len() as u64, + }); + match &self.core.compression { + Some(config) => { + let compress_started = self.elapsed(); + let compressed = match compress_payload(payload, config, MAX_DECOMPRESSED_BYTES) { + Ok(compressed) => compressed, + Err(error) => { + self.error_event(layer, ErrorKind::Compression, false); + return Err(Box::new(error)); + } + }; + payload = compressed.payload; + self.emit(Event::Compression { + labels: self.labels(layer), + outcome: compressed.outcome, + }); + if matches!( + compressed.outcome, + crate::observe::CompressionOutcome::Compressed + | crate::observe::CompressionOutcome::NotSmaller + ) { + self.emit(Event::CompressionDuration { + labels: self.labels(layer), + operation: CompressionOperation::Compress, + seconds: self.seconds_since(compress_started), + }); + } + if compressed.outcome == crate::observe::CompressionOutcome::Compressed { + self.emit(Event::CompressionRatio { + labels: self.labels(layer), + ratio: compressed.stored_bytes as f64 / compressed.original_bytes as f64, + }); + } + } + None => payload = escape_raw_payload(payload), + } + self.emit(Event::StoredSize { + labels: self.labels(layer), + bytes: payload.len() as u64, + }); + if let Some(allowed) = allowed { + if !allowed() { + return Ok(false); + } + } + let stamp = self.write_timestamp(layer)?; + if let Some(fence) = fence { + if stamp <= fence { + return Ok(false); + } + } + let mut ttl = if self.policy.stale_on_error_max_age_ms > 0 { + self.policy.stale_on_error_max_age_ms + } else { + self.policy.remote.ttl_ms + }; + if self.identity.tracked && ttl > MAX_TRACKED_VALUE_TTL_MS { + ttl = MAX_TRACKED_VALUE_TTL_MS; + self.error_event(layer, ErrorKind::TrackedTtlClamped, false); + } + let remote = self + .core + .remote + .clone() + .expect("remote layer requires an adapter"); + let request = WriteRequest { + value_key: self.keys.value.clone(), + frame: Frame { + created_at_ms: stamp, + payload, + }, + ttl_ms: ttl, + }; + let pending: Settled> = start_pending( + self.core.spawner.as_ref(), + async move { + remote + .write(request) + .await + .map_err(|e| Arc::from(e) as SharedError) + }, + |message| Err(Arc::new(PanicError(message)) as SharedError), + ); + match pending.wait().await { + Ok(()) => Ok(true), + Err(error) => { + self.error_event(layer, ErrorKind::CacheWrite, false); + Err(Box::new(SharedErrorWrapper(error))) + } + } + } + + pub(crate) fn can_recover(&self, error: &Error) -> bool { + let predicate = self + .op + .should_recover + .clone() + .or_else(|| self.core.should_recover.clone()); + match predicate { + None => error.is_fallback_timeout(), + Some(predicate) => match catch_unwind(AssertUnwindSafe(|| predicate(error))) { + Ok(allowed) => allowed, + Err(payload) => { + self.core + .log(LogEvent::RecoveryPredicateFailed(Box::new(PanicError( + panic_message(payload), + )))); + false + } + }, + } + } + + fn recovery_event(&self, outcome: RecoveryOutcome, age_ms: Option) { + self.emit(Event::StaleRecovery { + labels: self.labels.clone(), + outcome, + }); + if let Some(age) = age_ms { + self.emit(Event::StaleRecoveryValueAge { + labels: self.labels.clone(), + outcome, + seconds: age.max(0) as f64 / 1000.0, + }); + } + } + + /// Serve the retained stale candidate after an authorized source failure. + async fn recover(&self, frame: Option<&Frame>) -> Option { + let Some(frame) = frame else { + self.recovery_event(RecoveryOutcome::Miss, None); + return None; + }; + let max_age = self.policy.stale_on_error_max_age_ms; + let (age, valid) = self.frame_age(frame, Layer::Remote); + if !valid || age as u64 >= max_age { + self.recovery_event(RecoveryOutcome::Miss, None); + return None; + } + let value = match self.decode(frame, Layer::Remote).await { + Ok(value) => value, + Err(error) => { + self.core.log(LogEvent::RecoveryDecodeFailed(error)); + self.recovery_event(RecoveryOutcome::DeserializationError, None); + return None; + } + }; + let (age, valid) = self.frame_age(frame, Layer::Remote); + if !valid || age as u64 >= max_age { + self.recovery_event(RecoveryOutcome::Miss, None); + return None; + } + self.recovery_event(RecoveryOutcome::Served, Some(age)); + Some(value) + } +} diff --git a/rust/src/flight.rs b/rust/src/flight.rs new file mode 100644 index 00000000..e7ff87b7 --- /dev/null +++ b/rust/src/flight.rs @@ -0,0 +1,167 @@ +//! Settle-once cells shared by coalesced callers and detached raw work. + +use std::future::Future; +use std::panic::AssertUnwindSafe; +use std::pin::Pin; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll, Waker}; +use std::time::Duration; + +use futures::FutureExt; +use parking_lot::Mutex; + +use crate::error::Error; +use crate::local::StoredValue; +use crate::spawn::Spawner; + +struct SettledState { + value: Option, + wakers: Vec, +} + +/// A value that is set at most once and observed by any number of waiters. +pub(crate) struct Settled { + inner: Arc>>, +} + +impl Clone for Settled { + fn clone(&self) -> Self { + Settled { + inner: self.inner.clone(), + } + } +} + +impl Settled { + pub(crate) fn new() -> Self { + Settled { + inner: Arc::new(Mutex::new(SettledState { + value: None, + wakers: Vec::new(), + })), + } + } + + /// Store the value and wake every waiter. Returns false if already settled. + pub(crate) fn settle(&self, value: T) -> bool { + let wakers = { + let mut state = self.inner.lock(); + if state.value.is_some() { + return false; + } + state.value = Some(value); + std::mem::take(&mut state.wakers) + }; + for waker in wakers { + waker.wake(); + } + true + } + + pub(crate) fn peek(&self) -> Option { + self.inner.lock().value.clone() + } + + /// A future that completes with a clone of the settled value. + pub(crate) fn wait(&self) -> Wait { + Wait { + settled: self.clone(), + } + } +} + +pub(crate) struct Wait { + settled: Settled, +} + +impl Future for Wait { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut state = self.settled.inner.lock(); + if let Some(value) = &state.value { + return Poll::Ready(value.clone()); + } + if !state.wakers.iter().any(|w| w.will_wake(cx.waker())) { + state.wakers.push(cx.waker().clone()); + } + Poll::Pending + } +} + +impl Unpin for Wait {} + +/// Describe a panic payload. +pub(crate) fn panic_message(payload: Box) -> Arc { + if let Some(text) = payload.downcast_ref::<&str>() { + Arc::from(*text) + } else if let Some(text) = payload.downcast_ref::() { + Arc::from(text.as_str()) + } else { + Arc::from("non-string panic payload") + } +} + +/// Start `work` as detached raw work. The returned cell settles with its +/// result, or with `on_panic` when it panics. Callers that stop waiting keep +/// no ownership of the work. +pub(crate) fn start_pending( + spawner: &dyn Spawner, + work: F, + on_panic: impl FnOnce(Arc) -> T + Send + 'static, +) -> Settled +where + T: Clone + Send + 'static, + F: Future + Send + 'static, +{ + let settled = Settled::new(); + let cell = settled.clone(); + spawner.spawn(Box::pin(async move { + let outcome = match AssertUnwindSafe(work).catch_unwind().await { + Ok(value) => value, + Err(payload) => on_panic(panic_message(payload)), + }; + cell.settle(outcome); + })); + settled +} + +/// Complete once work that is already runnable has progressed (the executor's +/// deferred queue drained under a controlled scheduler). +pub(crate) async fn yield_deferred(spawner: &dyn Spawner) { + let done: Settled<()> = Settled::new(); + let cell = done.clone(); + spawner.defer(Box::pin(async move { + cell.settle(()); + })); + done.wait().await; +} + +/// Result of one execution shared by every coalesced caller. +pub(crate) type ValueResult = Result; + +/// One registered execution and its followers. +pub(crate) struct Flight { + pub(crate) result: Settled, + pub(crate) started: Duration, + pub(crate) followers: AtomicUsize, +} + +impl Flight { + pub(crate) fn new(started: Duration) -> Arc { + Arc::new(Flight { + result: Settled::new(), + started, + followers: AtomicUsize::new(0), + }) + } + + pub(crate) fn join(&self) { + self.followers.fetch_add(1, Ordering::Relaxed); + } + + pub(crate) fn followers(&self) -> usize { + self.followers.load(Ordering::Relaxed) + } +} diff --git a/rust/src/identity.rs b/rust/src/identity.rs index f232b9fb..7ab4a76f 100644 --- a/rust/src/identity.rs +++ b/rust/src/identity.rs @@ -22,7 +22,11 @@ pub struct Identity { } impl Identity { - pub fn new(key_type: impl Into, id: impl Into, use_case: impl Into) -> Self { + pub fn new( + key_type: impl Into, + id: impl Into, + use_case: impl Into, + ) -> Self { Identity { namespace: String::new(), key_type: key_type.into(), diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 44819196..17853add 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -12,27 +12,47 @@ pub mod cancel; pub mod clock; pub mod codec; +mod deadline; +mod engine; pub mod error; +mod execution; +mod flight; pub mod identity; pub mod limits; pub mod local; pub mod observe; +pub mod operation; pub mod policy; +pub mod preview; pub mod protocol; pub mod remote; +mod scope; +mod shadow; pub mod spawn; +mod use_case; pub use cancel::CancelToken; pub use clock::Clock; #[cfg(feature = "tokio")] pub use clock::SystemClock; pub use codec::{Codec, JsonCodec, Payload, SyncCodec}; +pub use engine::{ + CoalescingState, DialCache, DialCacheBuilder, PolicyProvider, ProcessCoalescingState, +}; pub use error::{BoxError, ConfigError, Error, FallbackTimeout, RemoteReadTimeout, SharedError}; pub use identity::{normalize_args, ArgValue, Identity, IdentityError, Keys}; pub use local::{LocalEntry, LocalStore, LruLocalStore, StoredValue}; pub use observe::{Event, Labels, LogEvent, LogLevel, Logger, Observer, ShadowMismatchDetails}; -pub use policy::{Policy, PolicyDefaults, PolicyError, ResolvedPolicy, RuntimePolicy, ShadowPolicy}; -pub use remote::{Frame, InvalidateRequest, MissReason, ReadContext, ReadRequest, ReadResult, Remote, WriteRequest}; +pub use operation::{Comparator, Operation, Preview, RecoveryPredicate, SourceBudget}; +pub use policy::{ + Policy, PolicyDefaults, PolicyError, ResolvedPolicy, RuntimePolicy, ShadowPolicy, +}; +pub use remote::{ + Frame, InvalidateRequest, MissReason, ReadContext, ReadRequest, ReadResult, Remote, + WriteRequest, +}; +pub use scope::Scope; pub use spawn::Spawner; #[cfg(feature = "tokio")] pub use spawn::TokioSpawner; +pub use use_case::{KeySpec, UseCase, UseCaseBuilder}; diff --git a/rust/src/local.rs b/rust/src/local.rs index 5770c1e7..9b5ef017 100644 --- a/rust/src/local.rs +++ b/rust/src/local.rs @@ -40,7 +40,9 @@ pub struct LruLocalStore { impl LruLocalStore { /// A store holding at most `capacity` entries. Capacity must be positive. pub fn new(capacity: NonZeroUsize) -> Self { - LruLocalStore { entries: lru::LruCache::new(capacity) } + LruLocalStore { + entries: lru::LruCache::new(capacity), + } } pub fn len(&self) -> usize { @@ -72,3 +74,21 @@ impl LocalStore for LruLocalStore { Ok(()) } } + +impl std::fmt::Debug for LocalEntry { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LocalEntry") + .field("inserted_ms", &self.inserted_ms) + .field("ttl_ms", &self.ttl_ms) + .finish_non_exhaustive() + } +} + +impl std::fmt::Debug for LruLocalStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LruLocalStore") + .field("len", &self.entries.len()) + .field("cap", &self.entries.cap()) + .finish() + } +} diff --git a/rust/src/observe.rs b/rust/src/observe.rs index 75255d1a..8e4bafce 100644 --- a/rust/src/observe.rs +++ b/rust/src/observe.rs @@ -137,27 +137,89 @@ pub struct OutcomeLabels { /// sizes report bytes. #[derive(Debug, Clone, PartialEq)] pub enum Event { - Request { labels: Labels }, - Miss { labels: Labels, reason: MissReason }, - Disabled { labels: Labels, reason: DisabledReason }, - Error { labels: Labels, error: ErrorKind, in_fallback: bool }, - Invalidation { namespace: Arc, key_type: Arc, layer: Layer }, - Coalesced { labels: OutcomeLabels, scope: CoalescingScope }, - ShadowValidation { labels: OutcomeLabels, outcome: ShadowOutcome }, + Request { + labels: Labels, + }, + Miss { + labels: Labels, + reason: MissReason, + }, + Disabled { + labels: Labels, + reason: DisabledReason, + }, + Error { + labels: Labels, + error: ErrorKind, + in_fallback: bool, + }, + Invalidation { + namespace: Arc, + key_type: Arc, + layer: Layer, + }, + Coalesced { + labels: OutcomeLabels, + scope: CoalescingScope, + }, + ShadowValidation { + labels: OutcomeLabels, + outcome: ShadowOutcome, + }, /// Age in seconds of the validated value at a match or confirmed mismatch, clamped at zero. - ShadowValueAge { labels: OutcomeLabels, outcome: ShadowOutcome, seconds: f64 }, + ShadowValueAge { + labels: OutcomeLabels, + outcome: ShadowOutcome, + seconds: f64, + }, /// Positive offset in seconds of a frame dated after the observing clock. - FutureTimestampOffset { labels: Labels, seconds: f64 }, - StaleRecovery { labels: OutcomeLabels, outcome: RecoveryOutcome }, - StaleRecoveryValueAge { labels: OutcomeLabels, outcome: RecoveryOutcome, seconds: f64 }, - Compression { labels: Labels, outcome: CompressionOutcome }, - Get { labels: Labels, seconds: f64 }, - Fallback { labels: Labels, seconds: f64 }, - Serialization { labels: Labels, operation: SerializationOperation, seconds: f64 }, - Size { labels: Labels, bytes: u64 }, - StoredSize { labels: Labels, bytes: u64 }, - CompressionRatio { labels: Labels, ratio: f64 }, - CompressionDuration { labels: Labels, operation: CompressionOperation, seconds: f64 }, + FutureTimestampOffset { + labels: Labels, + seconds: f64, + }, + StaleRecovery { + labels: OutcomeLabels, + outcome: RecoveryOutcome, + }, + StaleRecoveryValueAge { + labels: OutcomeLabels, + outcome: RecoveryOutcome, + seconds: f64, + }, + Compression { + labels: Labels, + outcome: CompressionOutcome, + }, + Get { + labels: Labels, + seconds: f64, + }, + Fallback { + labels: Labels, + seconds: f64, + }, + Serialization { + labels: Labels, + operation: SerializationOperation, + seconds: f64, + }, + Size { + labels: Labels, + bytes: u64, + }, + StoredSize { + labels: Labels, + bytes: u64, + }, + CompressionRatio { + labels: Labels, + ratio: f64, + }, + CompressionDuration { + labels: Labels, + operation: CompressionOperation, + seconds: f64, + }, } /// Receives every public diagnostic. Failures (returned errors or panics) diff --git a/rust/src/operation.rs b/rust/src/operation.rs new file mode 100644 index 00000000..578bd0c6 --- /dev/null +++ b/rust/src/operation.rs @@ -0,0 +1,244 @@ +//! Operations: the typed description of one cached call and its erased form. + +use std::any::Any; +use std::sync::Arc; + +use futures::future::BoxFuture; +use serde::de::DeserializeOwned; +use serde::Serialize; + +use crate::codec::{Codec, JsonCodec, Payload}; +use crate::error::{BoxError, Error}; +use crate::identity::Identity; +use crate::limits::DEFAULT_FALLBACK_TIMEOUT_MS; +use crate::local::StoredValue; +use crate::policy::Policy; +use crate::scope::Scope; + +/// The source deadline of one operation. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum SourceBudget { + /// The library default, 60,000 ms. + #[default] + Default, + /// No deadline: the caller waits for the source however long it takes. + /// Shadow work still uses the default budget. + Unbounded, + /// A deadline in whole milliseconds, `1..=2_147_483_647`. + Millis(u64), +} + +impl SourceBudget { + pub(crate) fn millis(self) -> Option { + match self { + SourceBudget::Default => Some(DEFAULT_FALLBACK_TIMEOUT_MS), + SourceBudget::Unbounded => None, + SourceBudget::Millis(ms) => Some(ms), + } + } +} + +/// Decides whether a source failure may be answered from the stale value +/// retained by the initial remote read. +pub type RecoveryPredicate = Arc bool + Send + Sync>; +/// Application equality used by shadow validation. +pub type Comparator = Arc bool + Send + Sync>; +/// Produces a bounded textual preview of a value for mismatch warnings. +pub type Preview = Arc Option + Send + Sync>; + +/// One inline cached call: its identity, static policy, budget and codecs. +/// +/// Registered use cases build this for every call; use it directly with +/// [`DialCache::get_or_load`](crate::DialCache::get_or_load) when a stable use +/// case is declared at the call site instead of being registered. +#[derive(Clone)] +pub struct Operation { + pub identity: Identity, + pub policy: Policy, + pub budget: SourceBudget, + pub codec: Arc>, + pub comparator: Comparator, + pub should_recover: Option, + pub preview: Option>, +} + +impl std::fmt::Debug for Operation { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Operation") + .field("identity", &self.identity) + .field("policy", &self.policy) + .field("budget", &self.budget) + .finish_non_exhaustive() + } +} + +impl Operation { + /// An operation using the JSON codec, `PartialEq` shadow comparison and + /// JSON mismatch previews. + pub fn new(identity: Identity) -> Self { + Operation { + identity, + policy: Policy::default(), + budget: SourceBudget::Default, + codec: Arc::new(JsonCodec), + comparator: Arc::new(|a: &T, b: &T| a == b), + should_recover: None, + preview: Some(Arc::new(|value: &T| serde_json::to_string(value).ok())), + } + } +} + +impl Operation { + /// An operation with an explicit codec and comparator, for values that + /// are not `serde` JSON values. Mismatch previews are absent unless set. + pub fn with_codec( + identity: Identity, + codec: Arc>, + comparator: impl Fn(&T, &T) -> bool + Send + Sync + 'static, + ) -> Self { + Operation { + identity, + policy: Policy::default(), + budget: SourceBudget::Default, + codec, + comparator: Arc::new(comparator), + should_recover: None, + preview: None, + } + } + + pub fn policy(mut self, policy: Policy) -> Self { + self.policy = policy; + self + } + + pub fn budget(mut self, budget: SourceBudget) -> Self { + self.budget = budget; + self + } + + pub fn should_recover( + mut self, + predicate: impl Fn(&Error) -> bool + Send + Sync + 'static, + ) -> Self { + self.should_recover = Some(Arc::new(predicate)); + self + } + + pub fn preview( + mut self, + preview: impl Fn(&T) -> Option + Send + Sync + 'static, + ) -> Self { + self.preview = Some(Arc::new(preview)); + self + } + + pub(crate) fn erase(self) -> ErasedParts { + let codec = self.codec; + let comparator = self.comparator; + let preview = self.preview; + ErasedParts { + identity: self.identity, + policy: self.policy, + budget: self.budget, + codec: Arc::new(TypedCodec { codec }), + compare: Arc::new(move |a: &StoredValue, b: &StoredValue| { + let (Some(a), Some(b)) = (a.downcast_ref::(), b.downcast_ref::()) else { + return Err("shadow comparison received a value of another type".into()); + }; + Ok(comparator(a, b)) + }), + should_recover: self.should_recover, + preview: preview.map(|preview| { + Arc::new(move |value: &StoredValue| { + value.downcast_ref::().and_then(|v| preview(v)) + }) as Arc Option + Send + Sync> + }), + } + } +} + +/// Source loader over erased values. +pub(crate) type ErasedLoad = + Arc BoxFuture<'static, Result> + Send + Sync>; +pub(crate) type ErasedCompare = + Arc Result + Send + Sync>; +pub(crate) type ErasedPreview = Arc Option + Send + Sync>; +pub(crate) type IdentityProvider = Arc Result + Send + Sync>; + +pub(crate) struct ErasedParts { + pub identity: Identity, + pub policy: Policy, + pub budget: SourceBudget, + pub codec: Arc, + pub compare: ErasedCompare, + pub should_recover: Option, + pub preview: Option, +} + +/// The type-erased operation the engine executes. +pub(crate) struct ErasedOperation { + pub identity: Identity, + pub identity_provider: Option, + pub policy: Policy, + pub budget: SourceBudget, + pub codec: Arc, + pub compare: ErasedCompare, + pub should_recover: Option, + pub preview: Option, + pub load: ErasedLoad, +} + +/// A codec over erased values. +pub(crate) trait ErasedCodec: Send + Sync { + fn encode<'a>(&'a self, value: &'a StoredValue) -> BoxFuture<'a, Result>; + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result>; +} + +struct TypedCodec { + codec: Arc>, +} + +impl ErasedCodec for TypedCodec { + fn encode<'a>(&'a self, value: &'a StoredValue) -> BoxFuture<'a, Result> { + match value.downcast_ref::() { + Some(value) => self.codec.encode(value), + None => Box::pin(std::future::ready(Err( + "codec received a value of another type".into(), + ))), + } + } + + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result> { + let decoding = self.codec.decode(payload); + Box::pin(async move { decoding.await.map(|value| Arc::new(value) as StoredValue) }) + } +} + +/// Erase a typed source closure. +pub(crate) fn erase_load(load: F) -> ErasedLoad +where + T: Send + Sync + 'static, + F: Fn(Scope) -> Fut + Send + Sync + 'static, + Fut: std::future::Future> + Send + 'static, +{ + Arc::new(move |scope: Scope| { + let future = load(scope); + Box::pin(async move { future.await.map(|value| Arc::new(value) as StoredValue) }) + }) +} + +/// Downcast an erased value to its typed form. +pub(crate) fn downcast_value( + value: StoredValue, +) -> Result, Error> { + Arc::downcast::(value).map_err(|_| { + Error::Config(crate::error::ConfigError::invalid(format!( + "cached value is not a {}", + std::any::type_name::() + ))) + }) +} + +#[allow(dead_code)] +fn _assert_any(_: &dyn Any) {} diff --git a/rust/src/policy.rs b/rust/src/policy.rs index 3af8fd9a..c348e174 100644 --- a/rust/src/policy.rs +++ b/rust/src/policy.rs @@ -55,7 +55,10 @@ impl Policy { stale_on_error_max_age_sec: Some(0), local_ramp: Some(0.0), remote_ramp: Some(0.0), - shadow: Some(ShadowPolicy { ramp: Some(0.0), log_mismatches: Some(false) }), + shadow: Some(ShadowPolicy { + ramp: Some(0.0), + log_mismatches: Some(false), + }), ..Policy::default() } } @@ -159,7 +162,9 @@ pub struct PolicyDefaults { impl Default for PolicyDefaults { fn default() -> Self { - PolicyDefaults { remote_read_timeout_ms: DEFAULT_REMOTE_READ_TIMEOUT_MS } + PolicyDefaults { + remote_read_timeout_ms: DEFAULT_REMOTE_READ_TIMEOUT_MS, + } } } @@ -217,6 +222,13 @@ pub fn resolve_policy( logical_key: &str, defaults: PolicyDefaults, ) -> Result { - let _ = (base, overlay, logical_key, defaults, MAX_CACHE_TTL_SEC, MAX_DEADLINE_MS); + let _ = ( + base, + overlay, + logical_key, + defaults, + MAX_CACHE_TTL_SEC, + MAX_DEADLINE_MS, + ); todo!("resolve_policy") } diff --git a/rust/src/preview.rs b/rust/src/preview.rs new file mode 100644 index 00000000..ce96b2e5 --- /dev/null +++ b/rust/src/preview.rs @@ -0,0 +1,46 @@ +//! Bounded previews for mismatch warnings. + +/// Maximum UTF-8 bytes of a logged cache key. +pub const SHADOW_LOG_KEY_MAX_BYTES: usize = 2 * 1024; +/// Maximum UTF-8 bytes of a logged value preview. +pub const SHADOW_LOG_VALUE_MAX_BYTES: usize = 8 * 1024; +/// Marker appended to a clamped preview. +pub const SHADOW_LOG_TRUNCATION_MARKER: &str = "...[truncated]"; + +/// Clamp text to `max_bytes` of UTF-8, ending a clamped result with the +/// truncation marker on a character boundary. +pub fn clamp_utf8(value: &str, max_bytes: usize) -> String { + if value.len() <= max_bytes { + return value.to_string(); + } + let mut limit = max_bytes.saturating_sub(SHADOW_LOG_TRUNCATION_MARKER.len()); + while limit > 0 && !value.is_char_boundary(limit) { + limit -= 1; + } + let mut result = String::with_capacity(limit + SHADOW_LOG_TRUNCATION_MARKER.len()); + result.push_str(&value[..limit]); + result.push_str(SHADOW_LOG_TRUNCATION_MARKER); + result +} + +pub fn preview_key(key: &str) -> String { + clamp_utf8(key, SHADOW_LOG_KEY_MAX_BYTES) +} + +pub fn preview_value(json: &str) -> String { + clamp_utf8(json, SHADOW_LOG_VALUE_MAX_BYTES) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn clamps_on_character_boundaries() { + let text = "é".repeat(2000); + let clamped = clamp_utf8(&text, 100); + assert!(clamped.len() <= 100); + assert!(clamped.ends_with(SHADOW_LOG_TRUNCATION_MARKER)); + assert_eq!(clamp_utf8("short", 100), "short"); + } +} diff --git a/rust/src/protocol/envelope.rs b/rust/src/protocol/envelope.rs index e17632f5..2a674d9e 100644 --- a/rust/src/protocol/envelope.rs +++ b/rust/src/protocol/envelope.rs @@ -1,7 +1,9 @@ //! Payload envelopes: escaping and zstd compression. use crate::codec::Payload; -use crate::limits::{DEFAULT_COMPRESSION_THRESHOLD_BYTES, DEFAULT_ZSTD_LEVEL, MAX_DECOMPRESSED_BYTES}; +use crate::limits::{ + DEFAULT_COMPRESSION_THRESHOLD_BYTES, DEFAULT_ZSTD_LEVEL, MAX_DECOMPRESSED_BYTES, +}; use crate::observe::CompressionOutcome; use super::frame::ProtocolError; @@ -21,7 +23,10 @@ pub struct CompressionConfig { impl Default for CompressionConfig { fn default() -> Self { - CompressionConfig { threshold_bytes: DEFAULT_COMPRESSION_THRESHOLD_BYTES, level: DEFAULT_ZSTD_LEVEL } + CompressionConfig { + threshold_bytes: DEFAULT_COMPRESSION_THRESHOLD_BYTES, + level: DEFAULT_ZSTD_LEVEL, + } } } @@ -65,7 +70,10 @@ pub fn compress_payload( } /// Reverse of [`compress_payload`], applied to every read. -pub fn decompress_payload(payload: Payload, max_decompressed_bytes: usize) -> CompressionReadResult { +pub fn decompress_payload( + payload: Payload, + max_decompressed_bytes: usize, +) -> CompressionReadResult { let _ = (payload, max_decompressed_bytes); todo!("decompress_payload") } diff --git a/rust/src/protocol/frame.rs b/rust/src/protocol/frame.rs index 64fec7f0..32ccebad 100644 --- a/rust/src/protocol/frame.rs +++ b/rust/src/protocol/frame.rs @@ -34,7 +34,11 @@ pub fn parse_watermark(raw: Option<&[u8]>) -> Option> /// Classify one raw read with the protocol's precedence: absent value, /// unsupported frame, tracked marker validity and zero stamp, fence, /// payload encoding, then a hit. -pub fn decode_frame(raw: Option<&[u8]>, tracked: bool, watermark: Option<&[u8]>) -> Result { +pub fn decode_frame( + raw: Option<&[u8]>, + tracked: bool, + watermark: Option<&[u8]>, +) -> Result { let _ = (raw, tracked, watermark); todo!("decode_frame") } diff --git a/rust/src/protocol/mod.rs b/rust/src/protocol/mod.rs index 5efffea8..68f47367 100644 --- a/rust/src/protocol/mod.rs +++ b/rust/src/protocol/mod.rs @@ -6,11 +6,12 @@ mod frame; mod text; pub use envelope::{ - compress_payload, decompress_payload, escape_raw_payload, CompressionConfig, CompressionReadResult, - CompressionWriteResult, MARKER_ESCAPED_RAW, MARKER_ZSTD_BINARY, MARKER_ZSTD_UTF8, + compress_payload, decompress_payload, escape_raw_payload, CompressionConfig, + CompressionReadResult, CompressionWriteResult, MARKER_ESCAPED_RAW, MARKER_ZSTD_BINARY, + MARKER_ZSTD_UTF8, }; pub use frame::{ - ceil_supported_cache_ttl_ms, decode_frame, encode_frame, normalize_read_result, parse_watermark, - read_result_from_untrusted_json, validate_timestamp_ms, ProtocolError, + ceil_supported_cache_ttl_ms, decode_frame, encode_frame, normalize_read_result, + parse_watermark, read_result_from_untrusted_json, validate_timestamp_ms, ProtocolError, }; pub use text::replacement_utf8; diff --git a/rust/src/remote.rs b/rust/src/remote.rs index 01a0d171..e7e0b0aa 100644 --- a/rust/src/remote.rs +++ b/rust/src/remote.rs @@ -66,7 +66,10 @@ pub enum ReadResult { impl ReadResult { pub fn miss(reason: MissReason) -> Self { - ReadResult::Miss { reason, observed_watermark_ms: None } + ReadResult::Miss { + reason, + observed_watermark_ms: None, + } } pub fn is_miss(&self) -> bool { @@ -120,7 +123,11 @@ pub struct InvalidateRequest { /// [`crate::protocol::decode_frame`] and [`crate::protocol::encode_frame`] /// for the wire format. pub trait Remote: Send + Sync + 'static { - fn read(&self, request: ReadRequest, context: ReadContext) -> BoxFuture<'_, Result>; + fn read( + &self, + request: ReadRequest, + context: ReadContext, + ) -> BoxFuture<'_, Result>; fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>>; fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>>; } diff --git a/rust/src/scope.rs b/rust/src/scope.rs new file mode 100644 index 00000000..9411cc53 --- /dev/null +++ b/rust/src/scope.rs @@ -0,0 +1,104 @@ +//! Enabled scopes and request memoization. + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use parking_lot::Mutex; + +use crate::flight::Flight; +use crate::local::StoredValue; + +pub(crate) struct OwnerState { + pub(crate) live: bool, + pub(crate) memo: HashMap, + pub(crate) flights: HashMap>, +} + +/// The outermost enabled scope's request state: memo and registered request flights. +pub(crate) struct Owner { + pub(crate) state: Mutex, +} + +impl Owner { + pub(crate) fn new() -> Arc { + Arc::new(Owner { + state: Mutex::new(OwnerState { + live: true, + memo: HashMap::new(), + flights: HashMap::new(), + }), + }) + } + + pub(crate) fn is_live(&self) -> bool { + self.state.lock().live + } + + pub(crate) fn close(&self) { + let mut state = self.state.lock(); + state.live = false; + state.memo.clear(); + state.flights.clear(); + } +} + +/// A caching scope handle. +/// +/// Caching is disabled by default. [`DialCache::enable`](crate::DialCache::enable) +/// opens the outermost enabled scope and hands its `Scope` to the callback; +/// pass it to every cached call made on behalf of that request. Nested +/// [`DialCache::enable_in`](crate::DialCache::enable_in) and +/// [`DialCache::disable_in`](crate::DialCache::disable_in) derive child +/// scopes that share the outer request memo. When the outermost callback +/// completes its scope closes: retained clones no longer enable caching and +/// late work cannot publish into the request memo. +/// +/// [`Scope::outside`] is the pass-through scope of code that runs on behalf of +/// no request: calls made with it invoke their source directly. +#[derive(Clone)] +pub struct Scope { + pub(crate) cache_id: u64, + pub(crate) owner: Option>, + pub(crate) enabled: bool, +} + +impl fmt::Debug for Scope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Scope") + .field("cache_id", &self.cache_id) + .field("enabled", &self.is_enabled()) + .field("has_owner", &self.owner.is_some()) + .finish() + } +} + +impl Scope { + /// The scope of work that runs on behalf of no request. Calls pass + /// straight through to their source. + pub fn outside() -> Scope { + Scope { + cache_id: 0, + owner: None, + enabled: false, + } + } + + /// Whether caching is enabled for calls made with this scope: it was + /// derived from an enabled scope and its outermost scope is still open. + pub fn is_enabled(&self) -> bool { + self.enabled && self.owner.as_ref().is_some_and(|owner| owner.is_live()) + } + + pub(crate) fn live_owner(&self) -> Option> { + self.owner.as_ref().filter(|owner| owner.is_live()).cloned() + } + + pub(crate) fn disabled_view(&self) -> Scope { + Scope { + cache_id: self.cache_id, + owner: self.owner.clone(), + enabled: false, + } + } +} diff --git a/rust/src/shadow.rs b/rust/src/shadow.rs new file mode 100644 index 00000000..d6d229e3 --- /dev/null +++ b/rust/src/shadow.rs @@ -0,0 +1,439 @@ +//! Detached shadow validation: dark reads and fills on ramped-down remote +//! serving, and served-hit re-validation against the source of truth. + +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use futures::future::{select, Either}; +use futures::FutureExt; +use parking_lot::Mutex; + +use crate::deadline::{await_deadline, since}; +use crate::error::Error; +use crate::execution::{Execution, RawRead}; +use crate::flight::{panic_message, start_pending, yield_deferred, Settled, ValueResult}; +use crate::limits::DEFAULT_FALLBACK_TIMEOUT_MS; +use crate::local::StoredValue; +use crate::observe::{ + ErrorKind, Event, Layer, LogEvent, MissReason, ShadowMismatchDetails, ShadowOutcome, +}; +use crate::preview::{preview_key, preview_value}; +use crate::remote::{Frame, ReadResult}; + +/// One admitted shadow job holding an instance slot. +pub(crate) struct ShadowFlight { + abandoned: AtomicBool, + stop: Settled<()>, +} + +impl ShadowFlight { + fn new() -> Arc { + Arc::new(ShadowFlight { + abandoned: AtomicBool::new(false), + stop: Settled::new(), + }) + } + + fn abandon(&self) { + self.abandoned.store(true, Ordering::SeqCst); + self.stop.settle(()); + } + + fn is_abandoned(&self) -> bool { + self.abandoned.load(Ordering::SeqCst) + } +} + +#[derive(Clone)] +struct Verdict { + outcome: ShadowOutcome, + age_ms: Option, + compared: Option<(StoredValue, StoredValue)>, +} + +impl Verdict { + fn of(outcome: ShadowOutcome) -> Verdict { + Verdict { + outcome, + age_ms: None, + compared: None, + } + } +} + +impl Execution { + /// Run the caller's source while a dark shadow job reads and may fill the + /// ramped-down remote layer without delaying the caller. + pub(crate) async fn dark_source( + self: &Arc, + layer: Layer, + local_miss: bool, + ) -> ValueResult { + let start = self.elapsed(); + let source: Settled = start_pending( + self.core.spawner.as_ref(), + { + let x = self.clone(); + async move { x.source(layer).await } + }, + |message| Err(Error::Panic(message)), + ); + self.schedule_shadow(None, Some(source.clone()), start); + let result = source.wait().await; + if let Ok(value) = &result { + if local_miss { + self.put_local(value.clone()); + } + } + result + } + + /// Admit one shadow job for this key if policy, the outcome hook and + /// instance capacity allow it. + pub(crate) fn schedule_shadow( + self: &Arc, + frame: Option, + source: Option>, + started: Duration, + ) { + let p = self.policy.shadow; + if p.config_error { + self.error_event(Layer::Remote, ErrorKind::ConfigResolution, false); + return; + } + if !p.enabled || !self.core.shadow_hook_enabled() { + return; + } + let flight = { + let mut state = self.core.state.lock(); + if state.shadows.contains_key(&self.keys.logical) + || state.shadows.len() >= self.core.shadow_max_in_flight + { + drop(state); + self.shadow_event(Verdict::of(ShadowOutcome::Dropped)); + return; + } + let flight = ShadowFlight::new(); + state + .shadows + .insert(self.keys.logical.clone(), flight.clone()); + flight + }; + if p.logging_config_error { + self.error_event(Layer::Remote, ErrorKind::ConfigResolution, false); + } + let x = self.clone(); + self.core.spawner.defer(Box::pin(async move { + x.run_shadow(flight, frame, source, started).await + })); + } + + fn shadow_event(&self, verdict: Verdict) { + self.emit(Event::ShadowValidation { + labels: self.labels.clone(), + outcome: verdict.outcome, + }); + if let Some(age) = verdict.age_ms { + self.emit(Event::ShadowValueAge { + labels: self.labels.clone(), + outcome: verdict.outcome, + seconds: age.max(0) as f64 / 1000.0, + }); + } + if verdict.outcome == ShadowOutcome::Mismatch && self.policy.shadow.log_mismatches { + let preview = |value: &StoredValue| -> Option { + let preview = self.op.preview.as_ref()?; + catch_unwind(AssertUnwindSafe(|| preview(value))) + .ok() + .flatten() + .map(|json| preview_value(&json)) + }; + let (cached, source) = match &verdict.compared { + Some((cached, source)) => (preview(cached), preview(source)), + None => (None, None), + }; + self.core + .log(LogEvent::ShadowMismatch(ShadowMismatchDetails { + namespace: self.labels.namespace.clone(), + use_case: self.labels.use_case.clone(), + key_type: self.labels.key_type.clone(), + cache_key: preview_key(&self.keys.logical), + cached_value_json: cached, + source_value_json: source, + })); + } + } + + async fn run_shadow( + self: Arc, + flight: Arc, + frame: Option, + source: Option>, + started: Duration, + ) { + let clock = self.core.clock.clone(); + let started = if source.is_none() { + clock.elapsed() + } else { + started + }; + let budget_ms = self + .op + .budget + .millis() + .unwrap_or(DEFAULT_FALLBACK_TIMEOUT_MS); + let reads: Arc>>> = Arc::new(Mutex::new(Vec::new())); + let validation: Settled = start_pending( + self.core.spawner.as_ref(), + { + let x = self.clone(); + let flight = flight.clone(); + let reads = reads.clone(); + async move { + let verdict = match AssertUnwindSafe(x.clone().validate( + flight.clone(), + frame, + source, + started, + budget_ms, + reads.clone(), + )) + .catch_unwind() + .await + { + Ok(verdict) => verdict, + Err(_) => Verdict::of(ShadowOutcome::Timeout), + }; + // Reads keep the slot after a read timeout; owned source, codec and + // write work already keeps this operation running until raw completion. + let pending = std::mem::take(&mut *reads.lock()); + let core = x.core.clone(); + let spawner = core.spawner.clone(); + let key = x.keys.logical.clone(); + spawner.spawn(Box::pin(async move { + for read in pending { + let _ = read.wait().await; + } + let mut state = core.state.lock(); + if state + .shadows + .get(&key) + .is_some_and(|f| Arc::ptr_eq(f, &flight)) + { + state.shadows.remove(&key); + } + })); + verdict + } + }, + |_| Verdict::of(ShadowOutcome::Timeout), + ); + let verdict = await_deadline( + clock.as_ref(), + &validation, + started, + Some(budget_ms), + || { + flight.abandon(); + Verdict::of(ShadowOutcome::Timeout) + }, + ) + .await; + self.shadow_event(verdict); + } + + async fn shadow_read( + &self, + reads: &Mutex>>, + max_age: bool, + retain_future: bool, + ) -> RawRead { + let begin = self.elapsed(); + self.emit(Event::Request { + labels: self.labels(Layer::RemoteShadow), + }); + let (bounded, raw) = self.raw_read(); + reads.lock().push(raw); + let result = bounded.await; + let result = match result { + Err(error) => { + let kind = if error + .downcast_ref::() + .is_some() + { + ErrorKind::CacheReadTimeout + } else { + ErrorKind::CacheRead + }; + self.error_event(Layer::RemoteShadow, kind, false); + Err(error) + } + Ok(mut read) => { + if let ReadResult::Hit(frame) = &read { + let (age, valid) = self.frame_age(frame, Layer::RemoteShadow); + if !valid && !(retain_future && age < 0) { + read = ReadResult::miss(MissReason::Unclassified); + } else if max_age && valid && age as u64 >= self.policy.remote.ttl_ms { + read = ReadResult::miss(MissReason::Expired); + } + } + if let ReadResult::Miss { reason, .. } = &read { + self.emit(Event::Miss { + labels: self.labels(Layer::RemoteShadow), + reason: *reason, + }); + } + Ok(read) + } + }; + self.emit(Event::Get { + labels: self.labels(Layer::RemoteShadow), + seconds: self.seconds_since(begin), + }); + result + } + + async fn validate( + self: Arc, + flight: Arc, + mut frame: Option, + source: Option>, + started: Duration, + budget_ms: u64, + reads: Arc>>>, + ) -> Verdict { + let clock = self.core.clock.clone(); + let budget = Duration::from_millis(budget_ms); + let expired = || { + if since(clock.as_ref(), started) >= budget { + flight.abandon(); + } + flight.is_abandoned() + }; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + let mut fill = false; + let mut fence: Option = None; + if source.is_some() { + let read = match self.shadow_read(&reads, true, false).await { + Ok(read) => read, + Err(_) => return Verdict::of(ShadowOutcome::RedisError), + }; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + match read { + ReadResult::Miss { + observed_watermark_ms, + .. + } => { + fill = true; + fence = observed_watermark_ms; + } + ReadResult::Hit(read_frame) => frame = Some(read_frame), + } + } + let value: ValueResult = match &source { + Some(source) => { + let result = match select(source.wait(), flight.stop.wait()).await { + Either::Left((result, _)) => result, + Either::Right(((), _)) => return Verdict::of(ShadowOutcome::Timeout), + }; + if result.is_ok() { + // Let the caller finish its own continuation before any shadow + // decode, comparison or dump work. + yield_deferred(self.core.spawner.as_ref()).await; + } + result + } + None => { + let disabled = self.scope.disabled_view(); + let future = (self.op.load)(disabled); + match AssertUnwindSafe(future).catch_unwind().await { + Ok(Ok(value)) => Ok(value), + Ok(Err(error)) => Err(Error::Source(Arc::from(error))), + Err(payload) => Err(Error::Panic(panic_message(payload))), + } + } + }; + let value = match value { + Ok(value) => value, + Err(_) => { + if source.is_some() && self.timed_out.load(Ordering::SeqCst) { + return Verdict::of(ShadowOutcome::Timeout); + } + return Verdict::of(ShadowOutcome::SourceError); + } + }; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + if fill { + let allowed = || !expired(); + let filled = self + .put_remote(&value, fence, Layer::RemoteShadow, Some(&allowed)) + .await; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + return match filled { + Ok(true) => Verdict::of(ShadowOutcome::Filled), + Ok(false) => Verdict::of(ShadowOutcome::FillFenced), + Err(error) => { + self.core.log(LogEvent::ShadowFillFailed(error)); + Verdict::of(ShadowOutcome::FillError) + } + }; + } + let Some(frame) = frame else { + return Verdict::of(ShadowOutcome::Timeout); + }; + let cached = match self.decode(&frame, Layer::RemoteShadow).await { + Ok(cached) => cached, + Err(_) => return Verdict::of(ShadowOutcome::DeserializationError), + }; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + let compare = self.op.compare.clone(); + let matches = match catch_unwind(AssertUnwindSafe(|| compare(&cached, &value))) { + Ok(Ok(matches)) => matches, + _ => return Verdict::of(ShadowOutcome::ComparisonError), + }; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + let age = self + .wall_ms() + .saturating_sub(frame.created_at_ms.min(i64::MAX as u64) as i64); + if matches { + return Verdict { + outcome: ShadowOutcome::Match, + age_ms: Some(age), + compared: None, + }; + } + let confirmation = match self.shadow_read(&reads, false, true).await { + Ok(confirmation) => confirmation, + Err(_) => return Verdict::of(ShadowOutcome::ConfirmationError), + }; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + match confirmation { + ReadResult::Hit(confirmed) if confirmed.payload.bytes == frame.payload.bytes => {} + _ => return Verdict::of(ShadowOutcome::Superseded), + } + let age = self + .wall_ms() + .saturating_sub(frame.created_at_ms.min(i64::MAX as u64) as i64); + Verdict { + outcome: ShadowOutcome::Mismatch, + age_ms: Some(age), + compared: Some((cached, value)), + } + } +} diff --git a/rust/src/use_case.rs b/rust/src/use_case.rs new file mode 100644 index 00000000..81408549 --- /dev/null +++ b/rust/src/use_case.rs @@ -0,0 +1,438 @@ +//! Registered, typed use cases. + +use std::future::Future; +use std::marker::PhantomData; +use std::sync::Arc; + +use futures::future::BoxFuture; +use serde::de::DeserializeOwned; +use serde::Serialize; + +use crate::codec::{Codec, JsonCodec}; +use crate::engine::DialCache; +use crate::error::{BoxError, ConfigError, Error}; +use crate::identity::{normalize_args, ArgValue, Identity}; +use crate::operation::{ + downcast_value, erase_load, Comparator, ErasedOperation, Operation, Preview, RecoveryPredicate, + SourceBudget, +}; +use crate::policy::Policy; +use crate::scope::Scope; + +/// The key of one call: the entity id and any secondary dimensions. +#[derive(Debug, Clone, PartialEq, Default)] +pub struct KeySpec { + pub id: String, + pub args: Vec<(String, ArgValue)>, +} + +impl KeySpec { + pub fn new(id: impl ToString) -> Self { + KeySpec { + id: id.to_string(), + args: Vec::new(), + } + } + + /// Add a secondary key dimension. Absent values are omitted; names are + /// sorted by UTF-16 code units when the key is built. + pub fn arg(mut self, name: impl Into, value: impl Into) -> Self { + self.args.push((name.into(), value.into())); + self + } +} + +impl From for KeySpec { + fn from(id: String) -> Self { + KeySpec::new(id) + } +} + +impl From<&str> for KeySpec { + fn from(id: &str) -> Self { + KeySpec::new(id) + } +} + +macro_rules! key_from_integer { + ($($t:ty),*) => { $(impl From<$t> for KeySpec { fn from(id: $t) -> Self { KeySpec::new(id) } })* }; +} +key_from_integer!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); + +impl From<&str> for ArgValue { + fn from(value: &str) -> Self { + ArgValue::Str(value.to_string()) + } +} + +impl From for ArgValue { + fn from(value: String) -> Self { + ArgValue::Str(value) + } +} + +impl From for ArgValue { + fn from(value: bool) -> Self { + ArgValue::Bool(value) + } +} + +impl From for ArgValue { + fn from(value: i64) -> Self { + ArgValue::Int(value) + } +} + +impl From for ArgValue { + fn from(value: i32) -> Self { + ArgValue::Int(value as i64) + } +} + +impl From for ArgValue { + fn from(value: u32) -> Self { + ArgValue::Int(value as i64) + } +} + +impl From for ArgValue { + fn from(value: f64) -> Self { + ArgValue::Number(value) + } +} + +impl> From> for ArgValue { + fn from(value: Option) -> Self { + match value { + Some(value) => value.into(), + None => ArgValue::Absent, + } + } +} + +type KeyFn = Arc KeySpec + Send + Sync>; +type SourceFn = + Arc BoxFuture<'static, Result> + Send + Sync>; + +/// Builds a registered use case. Finish with [`register`](Self::register) +/// for `serde` JSON values or [`register_custom`](Self::register_custom) +/// with an explicit codec and comparator. +pub struct UseCaseBuilder { + cache: DialCache, + key_type: String, + use_case: String, + tracked: bool, + policy: Policy, + budget: SourceBudget, + codec: Option>>, + comparator: Option>, + should_recover: Option, + preview: Option>, + key: Option>, + source: Option>, +} + +impl std::fmt::Debug for UseCaseBuilder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UseCaseBuilder") + .field("key_type", &self.key_type) + .field("use_case", &self.use_case) + .finish_non_exhaustive() + } +} + +impl UseCaseBuilder +where + Args: Clone + Send + Sync + 'static, + T: Send + Sync + 'static, +{ + pub(crate) fn new(cache: DialCache, key_type: String, use_case: String) -> Self { + UseCaseBuilder { + cache, + key_type, + use_case, + tracked: false, + policy: Policy::default(), + budget: SourceBudget::Default, + codec: None, + comparator: None, + should_recover: None, + preview: None, + key: None, + source: None, + } + } + + /// Static policy captured for every call of this use case. + pub fn policy(mut self, policy: Policy) -> Self { + self.policy = policy; + self + } + + /// Share the entity's invalidation watermark with every tracked variant. + pub fn tracked(mut self, tracked: bool) -> Self { + self.tracked = tracked; + self + } + + /// The source deadline. Defaults to 60 s. + pub fn budget(mut self, budget: SourceBudget) -> Self { + self.budget = budget; + self + } + + pub fn codec(mut self, codec: Arc>) -> Self { + self.codec = Some(codec); + self + } + + /// Application equality for shadow validation. Defaults to `PartialEq`. + pub fn comparator( + mut self, + comparator: impl Fn(&T, &T) -> bool + Send + Sync + 'static, + ) -> Self { + self.comparator = Some(Arc::new(comparator)); + self + } + + /// Override the instance classifier for stale recovery. + pub fn should_recover( + mut self, + predicate: impl Fn(&Error) -> bool + Send + Sync + 'static, + ) -> Self { + self.should_recover = Some(Arc::new(predicate)); + self + } + + /// Bounded textual preview of a value for mismatch warnings. + pub fn preview( + mut self, + preview: impl Fn(&T) -> Option + Send + Sync + 'static, + ) -> Self { + self.preview = Some(Arc::new(preview)); + self + } + + /// Select every input dimension that affects the value. Runs only for + /// enabled calls. + pub fn key(mut self, key: impl Fn(&Args) -> KeySpec + Send + Sync + 'static) -> Self { + self.key = Some(Arc::new(key)); + self + } + + /// The source of truth. It receives the call's scope so nested cached + /// calls can participate; served-hit shadow validation invokes it again + /// under disabled caching. + pub fn source(mut self, source: F) -> Self + where + F: Fn(Scope, Args) -> Fut + Send + Sync + 'static, + Fut: Future> + Send + 'static, + { + self.source = Some(Arc::new(move |scope, args| Box::pin(source(scope, args)))); + self + } + + fn finish( + self, + codec: Arc>, + comparator: Comparator, + preview: Option>, + ) -> Result, ConfigError> { + let key = self + .key + .ok_or_else(|| ConfigError::invalid("DialCache use case needs a key selector"))?; + let source = self + .source + .ok_or_else(|| ConfigError::invalid("DialCache use case needs a source"))?; + self.policy + .validate() + .map_err(|e| ConfigError::invalid(e.to_string()))?; + if let SourceBudget::Millis(ms) = self.budget { + if !(1..=crate::limits::MAX_DEADLINE_MS).contains(&ms) { + return Err(ConfigError::invalid( + "DialCache source budget is outside its domain", + )); + } + } + self.cache.register_use_case(&self.use_case)?; + Ok(UseCase { + inner: Arc::new(UseCaseInner { + cache: self.cache, + key_type: self.key_type, + use_case: self.use_case, + tracked: self.tracked, + policy: self.policy, + budget: self.budget, + codec, + comparator, + should_recover: self.should_recover, + preview, + key, + source, + }), + _args: PhantomData, + }) + } + + /// Register with an explicit codec and comparator (set with + /// [`codec`](Self::codec) and [`comparator`](Self::comparator)). + pub fn register_custom(self) -> Result, ConfigError> { + let codec = self + .codec + .clone() + .ok_or_else(|| ConfigError::invalid("DialCache use case needs a codec"))?; + let comparator = self + .comparator + .clone() + .ok_or_else(|| ConfigError::invalid("DialCache use case needs a comparator"))?; + let preview = self.preview.clone(); + self.finish(codec, comparator, preview) + } +} + +impl UseCaseBuilder +where + Args: Clone + Send + Sync + 'static, + T: Serialize + DeserializeOwned + PartialEq + Send + Sync + 'static, +{ + /// Register with the JSON codec, `PartialEq` comparison and JSON previews + /// unless overridden. + pub fn register(self) -> Result, ConfigError> { + let codec = self.codec.clone().unwrap_or_else(|| Arc::new(JsonCodec)); + let comparator = self + .comparator + .clone() + .unwrap_or_else(|| Arc::new(|a: &T, b: &T| a == b)); + let preview = self + .preview + .clone() + .or_else(|| Some(Arc::new(|value: &T| serde_json::to_string(value).ok()))); + self.finish(codec, comparator, preview) + } +} + +struct UseCaseInner { + cache: DialCache, + key_type: String, + use_case: String, + tracked: bool, + policy: Policy, + budget: SourceBudget, + codec: Arc>, + comparator: Comparator, + should_recover: Option, + preview: Option>, + key: KeyFn, + source: SourceFn, +} + +/// A registered use case: a typed cached function. +/// +/// Values are shared by reference and must be treated as immutable. +pub struct UseCase { + inner: Arc>, + _args: PhantomData, +} + +impl Clone for UseCase { + fn clone(&self) -> Self { + UseCase { + inner: self.inner.clone(), + _args: PhantomData, + } + } +} + +impl std::fmt::Debug for UseCase { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("UseCase") + .field("key_type", &self.inner.key_type) + .field("use_case", &self.inner.use_case) + .finish() + } +} + +impl UseCase +where + Args: Clone + Send + Sync + 'static, + T: Send + Sync + 'static, +{ + pub fn use_case(&self) -> &str { + &self.inner.use_case + } + + pub fn key_type(&self) -> &str { + &self.inner.key_type + } + + /// Get the value for `args` through the configured layers. + pub async fn get(&self, scope: &Scope, args: Args) -> Result, Error> { + let inner = self.inner.clone(); + let identity = Identity { + namespace: inner.cache.core.namespace.to_string(), + key_type: inner.key_type.clone(), + id: String::new(), + use_case: inner.use_case.clone(), + tracked: inner.tracked, + args: Vec::new(), + }; + let parts = Operation { + identity: identity.clone(), + policy: inner.policy.clone(), + budget: inner.budget, + codec: inner.codec.clone(), + comparator: inner.comparator.clone(), + should_recover: inner.should_recover.clone(), + preview: inner.preview.clone(), + } + .erase(); + let provider_args = args.clone(); + let provider_inner = inner.clone(); + let identity_provider = Arc::new(move || -> Result { + let spec = (provider_inner.key)(&provider_args); + let args = normalize_args(spec.args.clone())?; + Ok(Identity { + id: spec.id, + args, + ..identity.clone() + }) + }); + let source_inner = inner.clone(); + let load = erase_load(move |scope: Scope| (source_inner.source)(scope, args.clone())); + let erased = ErasedOperation { + identity: parts.identity, + identity_provider: Some(identity_provider), + policy: parts.policy, + budget: parts.budget, + codec: parts.codec, + compare: parts.compare, + should_recover: parts.should_recover, + preview: parts.preview, + load, + }; + let value = inner.cache.execute(scope, erased).await?; + downcast_value::(value) + } + + /// Invoke the source directly, outside any scope. Equivalent to + /// [`get`](Self::get) with [`Scope::outside`]. + pub async fn get_uncached(&self, args: Args) -> Result, Error> { + self.get(&Scope::outside(), args).await + } +} + +impl DialCache { + /// Start registering a use case. `key_type` names the entity type and + /// `use_case` the operation; the pair identifies cached values. + pub fn use_case( + &self, + key_type: impl Into, + use_case: impl Into, + ) -> UseCaseBuilder + where + Args: Clone + Send + Sync + 'static, + T: Send + Sync + 'static, + { + UseCaseBuilder::new(self.clone(), key_type.into(), use_case.into()) + } +} From 122b7cf370178af5f21c326288e05d21c65d9e82 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 00:25:39 -0700 Subject: [PATCH 06/31] feat(rust): implement version-1 frames, replacement UTF-8 text and the compression envelope Port the portable wire protocol to the Rust crate: frame encoding and the decode classification precedence, watermark parsing, the semantic read-result trust boundary (untrusted JSON replies and normalization), the timestamp and duration domains, WHATWG replacement UTF-8 conversion, and the payload envelope (escaping, zstd compression selection and first-frame decompression with the output cap). Add a vector runner for the eight frame/decode/timestamp/duration/envelope groups and a conformance test that replays every fixed and Quint-generated protocol vector (985 in total), plus module unit tests for the PROTOCOL.md replacement table, first-frame trailer handling, truncated bodies and the read/write limit outcomes. The test loads fixtures through a temporary local copy of the shared fixture helpers (tests/formal/fixtures_tmp.rs) until the shared tests/formal/fixtures.rs lands. --- rust/src/protocol/envelope.rs | 516 ++++++++++++++++++++++++++++- rust/src/protocol/frame.rs | 515 +++++++++++++++++++++++++++- rust/src/protocol/text.rs | 126 ++++++- rust/tests/formal/fixtures_tmp.rs | 169 ++++++++++ rust/tests/formal/frame_vectors.rs | 388 ++++++++++++++++++++++ rust/tests/protocol_frames.rs | 85 +++++ 6 files changed, 1770 insertions(+), 29 deletions(-) create mode 100644 rust/tests/formal/fixtures_tmp.rs create mode 100644 rust/tests/formal/frame_vectors.rs create mode 100644 rust/tests/protocol_frames.rs diff --git a/rust/src/protocol/envelope.rs b/rust/src/protocol/envelope.rs index e17632f5..014bac0a 100644 --- a/rust/src/protocol/envelope.rs +++ b/rust/src/protocol/envelope.rs @@ -1,15 +1,26 @@ //! Payload envelopes: escaping and zstd compression. +//! +//! Byte 0 of a binary payload written by this version or later selects the +//! envelope: `0x00` escapes raw serializer output whose own first byte would +//! collide, `0x01` marks a zstd frame of UTF-8 text and `0x02` a zstd frame of +//! binary bytes. Raw output is escaped on every write, and every read +//! interprets the envelope, so disabling compression never strands entries. +//! See "Envelope selection and codec environment" in `formal/PROTOCOL.md`. use crate::codec::Payload; -use crate::limits::{DEFAULT_COMPRESSION_THRESHOLD_BYTES, DEFAULT_ZSTD_LEVEL, MAX_DECOMPRESSED_BYTES}; +use crate::limits::{DEFAULT_COMPRESSION_THRESHOLD_BYTES, DEFAULT_ZSTD_LEVEL, MAX_SAFE_INTEGER}; use crate::observe::CompressionOutcome; use super::frame::ProtocolError; +use super::text::replacement_utf8; pub const MARKER_ESCAPED_RAW: u8 = 0x00; pub const MARKER_ZSTD_UTF8: u8 = 0x01; pub const MARKER_ZSTD_BINARY: u8 = 0x02; +const MIN_ZSTD_LEVEL: i32 = 1; +const MAX_ZSTD_LEVEL: i32 = 22; + /// Write-side compression policy. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct CompressionConfig { @@ -21,14 +32,27 @@ pub struct CompressionConfig { impl Default for CompressionConfig { fn default() -> Self { - CompressionConfig { threshold_bytes: DEFAULT_COMPRESSION_THRESHOLD_BYTES, level: DEFAULT_ZSTD_LEVEL } + CompressionConfig { + threshold_bytes: DEFAULT_COMPRESSION_THRESHOLD_BYTES, + level: DEFAULT_ZSTD_LEVEL, + } } } impl CompressionConfig { + /// The threshold must be a positive safe integer and the level `1..=22`. pub fn validate(&self) -> Result<(), ProtocolError> { - let _ = MAX_DECOMPRESSED_BYTES; - todo!("CompressionConfig::validate") + if self.threshold_bytes < 1 || self.threshold_bytes as u64 > MAX_SAFE_INTEGER { + return Err(ProtocolError::Compression( + "threshold_bytes must be a positive safe integer".to_string(), + )); + } + if !(MIN_ZSTD_LEVEL..=MAX_ZSTD_LEVEL).contains(&self.level) { + return Err(ProtocolError::Compression( + "level must be an integer between 1 and 22".to_string(), + )); + } + Ok(()) } } @@ -49,23 +73,493 @@ pub struct CompressionReadResult { /// Prefix raw binary output whose first byte would collide with the envelope. pub fn escape_raw_payload(payload: Payload) -> Payload { - let _ = payload; - todo!("escape_raw_payload") + if payload.binary + && payload + .bytes + .first() + .is_some_and(|first| *first <= MARKER_ZSTD_BINARY) + { + let mut escaped = Vec::with_capacity(payload.bytes.len() + 1); + escaped.push(MARKER_ESCAPED_RAW); + escaped.extend_from_slice(&payload.bytes); + return Payload::binary(escaped); + } + payload } /// Compress when the payload meets the threshold and the marked result is /// strictly smaller than the escaped raw form. +/// +/// Text payloads receive replacement UTF-8 conversion first, so the measured +/// size is the stored text's. The threshold comparison precedes the output +/// cap: a value the read side would refuse is stored raw (`write_over_limit`) +/// and subject to Redis's own value limit instead. `original_bytes` is the +/// payload's own length and `stored_bytes` the length actually written. pub fn compress_payload( payload: Payload, config: &CompressionConfig, max_decompressed_bytes: usize, ) -> Result { - let _ = (payload, config, max_decompressed_bytes); - todo!("compress_payload") + config.validate()?; + let payload = if payload.binary { + payload + } else { + Payload { + bytes: replacement_utf8(&payload.bytes), + binary: false, + } + }; + let original_bytes = payload.len(); + let raw = escape_raw_payload(payload.clone()); + let mut result = CompressionWriteResult { + stored_bytes: raw.len(), + payload: raw, + outcome: CompressionOutcome::BelowThreshold, + original_bytes, + }; + if original_bytes < config.threshold_bytes { + return Ok(result); + } + if original_bytes > max_decompressed_bytes { + result.outcome = CompressionOutcome::WriteOverLimit; + return Ok(result); + } + let compressed = zstd_compress(&payload.bytes, config.level)?; + if compressed.len() + 1 >= result.stored_bytes { + result.outcome = CompressionOutcome::NotSmaller; + return Ok(result); + } + let mut marked = Vec::with_capacity(compressed.len() + 1); + marked.push(if payload.binary { + MARKER_ZSTD_BINARY + } else { + MARKER_ZSTD_UTF8 + }); + marked.extend_from_slice(&compressed); + result.stored_bytes = marked.len(); + result.payload = Payload::binary(marked); + result.outcome = CompressionOutcome::Compressed; + Ok(result) } /// Reverse of [`compress_payload`], applied to every read. -pub fn decompress_payload(payload: Payload, max_decompressed_bytes: usize) -> CompressionReadResult { - let _ = (payload, max_decompressed_bytes); - todo!("decompress_payload") +/// +/// Text payloads and empty binary payloads pass through. An escape prefix is +/// removed only when the next byte is one the writer escapes; other `0x00` +/// leads and unknown markers pass through unchanged. A marked payload decodes +/// only its first zstd frame and ignores trailing bytes, like Node's +/// synchronous decoder. Malformed or truncated zstd data keeps the original +/// marked bytes with `fallback_raw`; output beyond `max_decompressed_bytes` +/// (or a frame demanding more decoder memory than allowed) keeps them with +/// `read_over_limit`. Successful `0x01` output receives replacement UTF-8 +/// conversion; `0x02` output is exact bytes. The input is never mutated, so +/// repeated loads of a retained payload are independent. +pub fn decompress_payload( + payload: Payload, + max_decompressed_bytes: usize, +) -> CompressionReadResult { + let passthrough = |payload| CompressionReadResult { + payload, + outcome: None, + }; + if !payload.binary { + return passthrough(payload); + } + let Some(&marker) = payload.bytes.first() else { + return passthrough(payload); + }; + match marker { + MARKER_ESCAPED_RAW => { + if payload + .bytes + .get(1) + .is_some_and(|next| *next <= MARKER_ZSTD_BINARY) + { + return passthrough(Payload::binary(&payload.bytes[1..])); + } + passthrough(payload) + } + MARKER_ZSTD_UTF8 | MARKER_ZSTD_BINARY => { + match decode_first_zstd_frame(&payload.bytes[1..], max_decompressed_bytes) { + Ok(decoded) => { + let decoded = if marker == MARKER_ZSTD_UTF8 { + Payload { + bytes: replacement_utf8(&decoded), + binary: false, + } + } else { + Payload::binary(decoded) + }; + CompressionReadResult { + payload: decoded, + outcome: Some(CompressionOutcome::Decompressed), + } + } + Err(outcome) => CompressionReadResult { + payload, + outcome: Some(outcome), + }, + } + } + _ => passthrough(payload), + } +} + +/// One zstd frame of `bytes` at `level`, without a checksum, as the writer +/// stores it (single-shot, so the frame header carries the content size). +fn zstd_compress(bytes: &[u8], level: i32) -> Result, ProtocolError> { + let codec = + |code: usize| ProtocolError::Compression(zstd_safe::get_error_name(code).to_string()); + let mut context = zstd_safe::CCtx::create(); + context + .set_parameter(zstd_safe::CParameter::CompressionLevel(level)) + .map_err(codec)?; + context + .set_parameter(zstd_safe::CParameter::ChecksumFlag(false)) + .map_err(codec)?; + let mut out = vec![0u8; zstd_safe::compress_bound(bytes.len())]; + let written = context.compress2(&mut out[..], bytes).map_err(codec)?; + out.truncate(written); + Ok(out) +} + +/// Decode only the first zstd frame of `body`, ignoring any trailer, with the +/// output cap enforced while streaming so a small stored frame can never force +/// a large allocation. Skippable frames decode to empty output. +fn decode_first_zstd_frame( + body: &[u8], + max_decompressed_bytes: usize, +) -> Result, CompressionOutcome> { + // Rejects truncated and malformed headers or block sequences up front, + // like Go's firstZstdFrame; only a complete first frame is decoded. + let frame_len = + zstd_safe::find_frame_compressed_size(body).map_err(|_| CompressionOutcome::FallbackRaw)?; + let frame = body + .get(..frame_len) + .ok_or(CompressionOutcome::FallbackRaw)?; + + let mut context = zstd_safe::DCtx::create(); + let mut out = Vec::new(); + let mut input = zstd_safe::InBuffer::around(frame); + // Each step writes at most one chunk; a cap smaller than a chunk still + // only needs one chunk to observe the overflow. + let chunk_len = zstd_safe::DCtx::out_size().max(1); + let mut chunk = vec![0u8; chunk_len]; + loop { + let mut output = zstd_safe::OutBuffer::around(&mut chunk[..]); + let consumed_before = input.pos(); + let hint = context + .decompress_stream(&mut output, &mut input) + .map_err(classify_decoder_error)?; + let produced = output.as_slice(); + if out.len() + produced.len() > max_decompressed_bytes { + return Err(CompressionOutcome::ReadOverLimit); + } + out.extend_from_slice(produced); + if hint == 0 { + return Ok(out); + } + if input.pos() == frame.len() && produced.is_empty() { + // The verified frame ended without the decoder finishing it. + return Err(CompressionOutcome::FallbackRaw); + } + if input.pos() == consumed_before && produced.is_empty() { + return Err(CompressionOutcome::FallbackRaw); + } + } +} + +/// zstd errors that mean the frame exceeds the decoder's resource limits are +/// read-limit outcomes; anything else falls back to the raw bytes. +fn classify_decoder_error(code: usize) -> CompressionOutcome { + let name = zstd_safe::get_error_name(code); + if name.contains("too much memory") || name.contains("Destination buffer is too small") { + CompressionOutcome::ReadOverLimit + } else { + CompressionOutcome::FallbackRaw + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::limits::MAX_DECOMPRESSED_BYTES; + + fn text(len: usize) -> Payload { + Payload::text("x".repeat(len)) + } + + fn compressed(payload: Payload) -> CompressionWriteResult { + let result = compress_payload( + payload, + &CompressionConfig { + threshold_bytes: 1, + level: 3, + }, + MAX_DECOMPRESSED_BYTES, + ) + .unwrap(); + assert_eq!(result.outcome, CompressionOutcome::Compressed); + result + } + + #[test] + fn config_validation() { + assert_eq!(CompressionConfig::default().validate(), Ok(())); + assert!(CompressionConfig { + threshold_bytes: 0, + level: 3 + } + .validate() + .is_err()); + assert!(CompressionConfig { + threshold_bytes: usize::MAX, + level: 3 + } + .validate() + .is_err()); + assert!(CompressionConfig { + threshold_bytes: 1, + level: 0 + } + .validate() + .is_err()); + assert!(CompressionConfig { + threshold_bytes: 1, + level: 23 + } + .validate() + .is_err()); + assert_eq!( + CompressionConfig { + threshold_bytes: 1, + level: 22 + } + .validate(), + Ok(()) + ); + let invalid = compress_payload( + text(1), + &CompressionConfig { + threshold_bytes: 1, + level: 23, + }, + MAX_DECOMPRESSED_BYTES, + ); + assert!(matches!(invalid, Err(ProtocolError::Compression(_)))); + } + + #[test] + fn escaping_and_passthrough() { + assert_eq!( + escape_raw_payload(Payload::binary(vec![0, 9])).bytes, + [0, 0, 9] + ); + assert_eq!(escape_raw_payload(Payload::binary(vec![2])).bytes, [0, 2]); + assert_eq!(escape_raw_payload(Payload::binary(vec![3])).bytes, [3]); + assert_eq!( + escape_raw_payload(Payload::binary(Vec::new())).bytes, + Vec::::new() + ); + assert_eq!(escape_raw_payload(Payload::text("\u{0}")).bytes, [0]); + // Only a prefix the writer produces is stripped; legacy 0x00 leads stay. + assert_eq!( + decompress_payload(Payload::binary(vec![0, 1, 7]), 10), + CompressionReadResult { + payload: Payload::binary(vec![1, 7]), + outcome: None + } + ); + for untouched in [ + Payload::binary(vec![0, 3]), + Payload::binary(vec![0]), + Payload::binary(vec![3, 1]), + Payload::binary(Vec::new()), + Payload::text("\u{1}x"), + ] { + assert_eq!( + decompress_payload(untouched.clone(), 10), + CompressionReadResult { + payload: untouched, + outcome: None + } + ); + } + } + + /// Port of go/codec_test.go TestCompressionLimitsAndIndependentLoads. + #[test] + fn compression_limits_and_independent_loads() { + let raw = text(4096); + let written = compressed(raw.clone()); + let too_large = decompress_payload(written.payload.clone(), 4095); + assert_eq!(too_large.outcome, Some(CompressionOutcome::ReadOverLimit)); + assert_eq!(too_large.payload, written.payload); + let exact = decompress_payload(written.payload.clone(), 4096); + assert_eq!( + exact, + CompressionReadResult { + payload: raw.clone(), + outcome: Some(CompressionOutcome::Decompressed) + } + ); + let refused = compress_payload( + raw.clone(), + &CompressionConfig { + threshold_bytes: 1, + level: 3, + }, + 4095, + ) + .unwrap(); + assert_eq!(refused.outcome, CompressionOutcome::WriteOverLimit); + assert_eq!(refused.payload, raw); + assert_eq!((refused.original_bytes, refused.stored_bytes), (4096, 4096)); + // Repeated loads of a retained input are independent. + let mut first = decompress_payload(written.payload.clone(), MAX_DECOMPRESSED_BYTES); + first.payload.bytes[0] = b'y'; + assert_eq!( + decompress_payload(written.payload.clone(), MAX_DECOMPRESSED_BYTES).payload, + raw + ); + let escaped = escape_raw_payload(Payload::binary(vec![1, 2, 3])); + let mut first = decompress_payload(escaped.clone(), MAX_DECOMPRESSED_BYTES); + first.payload.bytes[0] = 9; + assert_eq!( + decompress_payload(escaped, MAX_DECOMPRESSED_BYTES) + .payload + .bytes, + [1, 2, 3] + ); + // A zero cap admits only empty output. + assert_eq!( + decompress_payload(written.payload.clone(), 0).outcome, + Some(CompressionOutcome::ReadOverLimit) + ); + } + + /// Port of go/codec_test.go TestZstdFirstFrameAndMalformedBodies. + #[test] + fn first_frame_and_malformed_bodies() { + let source = Payload::text("first".repeat(100)); + let written = compressed(source.clone()); + for trailer in [&b"CRC!"[..], &written.payload.bytes[1..]] { + let mut marked = written.payload.bytes.clone(); + marked.extend_from_slice(trailer); + let result = decompress_payload(Payload::binary(marked), MAX_DECOMPRESSED_BYTES); + assert_eq!( + result, + CompressionReadResult { + payload: source.clone(), + outcome: Some(CompressionOutcome::Decompressed) + } + ); + } + let half = written.payload.bytes[..written.payload.bytes.len() / 2].to_vec(); + for marked in [ + vec![1], + vec![2], + half, + vec![1, 0x28, 0xB5, 0x2F, 0xFD], + vec![2, 0xFF, 0xFF], + ] { + let got = decompress_payload(Payload::binary(marked.clone()), MAX_DECOMPRESSED_BYTES); + assert_eq!( + got, + CompressionReadResult { + payload: Payload::binary(marked), + outcome: Some(CompressionOutcome::FallbackRaw) + } + ); + } + } + + #[test] + fn skippable_frames_decode_to_empty_output() { + // Magic 0x184D2A50, 4-byte little-endian size, then that many bytes. + let skippable = [ + 0x01, 0x50, 0x2A, 0x4D, 0x18, 0x02, 0x00, 0x00, 0x00, 0xAA, 0xBB, 0xCC, + ]; + let got = decompress_payload(Payload::binary(skippable.to_vec()), MAX_DECOMPRESSED_BYTES); + assert_eq!( + got, + CompressionReadResult { + payload: Payload::text(""), + outcome: Some(CompressionOutcome::Decompressed) + } + ); + // Zero cap with empty output is still a successful decode. + assert_eq!( + decompress_payload(Payload::binary(skippable.to_vec()), 0).outcome, + Some(CompressionOutcome::Decompressed) + ); + let truncated = skippable[..8].to_vec(); + assert_eq!( + decompress_payload(Payload::binary(truncated), MAX_DECOMPRESSED_BYTES).outcome, + Some(CompressionOutcome::FallbackRaw) + ); + } + + #[test] + fn selection_rule_and_types() { + let config = CompressionConfig { + threshold_bytes: 4, + level: 3, + }; + let below = compress_payload(text(3), &config, MAX_DECOMPRESSED_BYTES).unwrap(); + assert_eq!( + (below.outcome, below.original_bytes, below.stored_bytes), + (CompressionOutcome::BelowThreshold, 3, 3) + ); + let incompressible = compress_payload( + Payload::binary(vec![1, 2, 3, 4]), + &config, + MAX_DECOMPRESSED_BYTES, + ) + .unwrap(); + assert_eq!(incompressible.outcome, CompressionOutcome::NotSmaller); + assert_eq!(incompressible.payload.bytes, [0, 1, 2, 3, 4]); + assert_eq!( + (incompressible.original_bytes, incompressible.stored_bytes), + (4, 5) + ); + let binary = compressed(Payload::binary(vec![7; 500])); + assert_eq!(binary.payload.bytes[0], MARKER_ZSTD_BINARY); + assert!(binary.payload.binary && binary.stored_bytes < 500); + assert_eq!( + decompress_payload(binary.payload, MAX_DECOMPRESSED_BYTES).payload, + Payload::binary(vec![7; 500]) + ); + let text_result = compressed(text(500)); + assert_eq!(text_result.payload.bytes[0], MARKER_ZSTD_UTF8); + assert_eq!( + decompress_payload(text_result.payload, MAX_DECOMPRESSED_BYTES).payload, + text(500) + ); + // Ill-formed text is replaced before measuring and compressing. + let ill_formed = Payload { + bytes: vec![0xE2, 0x82, b'A'], + binary: false, + }; + let stored = compress_payload( + ill_formed, + &CompressionConfig { + threshold_bytes: 100, + level: 3, + }, + MAX_DECOMPRESSED_BYTES, + ) + .unwrap(); + assert_eq!(stored.payload, Payload::text("\u{FFFD}A")); + assert_eq!((stored.original_bytes, stored.stored_bytes), (4, 4)); + // Decoded 0x01 text is replaced; 0x02 bytes are exact. + let mut marked_text = + compressed(Payload::binary([0xE2, 0x82, b'A', 0xFF].repeat(50))).payload; + marked_text.bytes[0] = MARKER_ZSTD_UTF8; + let decoded = decompress_payload(marked_text, MAX_DECOMPRESSED_BYTES); + assert_eq!( + decoded.payload, + Payload::text("\u{FFFD}A\u{FFFD}".repeat(50)) + ); + } } diff --git a/rust/src/protocol/frame.rs b/rust/src/protocol/frame.rs index 64fec7f0..adff42d8 100644 --- a/rust/src/protocol/frame.rs +++ b/rust/src/protocol/frame.rs @@ -1,6 +1,15 @@ //! Version-1 frames and semantic read results. +//! +//! A frame is `[0x01] ++ big-endian u64 createdAtMs ++ [0x00 text | 0x01 binary] +//! ++ payload`. [`decode_frame`] applies the protocol's classification +//! precedence to one raw read; [`normalize_read_result`] is the trust boundary +//! above wire decoding that every semantic adapter result passes through. -use crate::remote::{Frame, ReadResult}; +use crate::codec::Payload; +use crate::limits::{MAX_SAFE_INTEGER, MAX_SUPPORTED_DURATION_MS}; +use crate::remote::{Frame, MissReason, ReadResult}; + +use super::text::replacement_utf8; /// Wire-level failures that are errors rather than misses. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] @@ -17,51 +26,525 @@ pub enum ProtocolError { Compression(String), } +const FRAME_VERSION: u8 = 1; +const FRAME_HEADER_LEN: usize = 10; +const ENCODING_TEXT: u8 = 0; +const ENCODING_BINARY: u8 = 1; + /// Encode a frame: version byte, big-endian uint64 stamp, encoding tag, payload. /// Text payloads receive replacement UTF-8 conversion first. pub fn encode_frame(frame: &Frame) -> Result, ProtocolError> { - let _ = frame; - todo!("encode_frame") + if frame.created_at_ms > MAX_SAFE_INTEGER { + return Err(ProtocolError::InvalidTimestamp); + } + let mut out = Vec::with_capacity(FRAME_HEADER_LEN + frame.payload.len()); + out.push(FRAME_VERSION); + out.extend_from_slice(&frame.created_at_ms.to_be_bytes()); + if frame.payload.binary { + out.push(ENCODING_BINARY); + out.extend_from_slice(&frame.payload.bytes); + } else { + out.push(ENCODING_TEXT); + out.extend_from_slice(&replacement_utf8(&frame.payload.bytes)); + } + Ok(out) } /// Parse a watermark bulk string. `None` input is absent (a valid zero /// baseline); `Some(Err)` is malformed; `Some(Ok(value))` is a valid stamp. +/// +/// Only ASCII digits are accepted. Leading zeros are ignored even when the +/// literal would overflow a machine integer; the remaining digits must fit +/// the safe-integer domain. pub fn parse_watermark(raw: Option<&[u8]>) -> Option> { - let _ = raw; - todo!("parse_watermark") + let raw = raw?; + Some(parse_watermark_digits(raw)) +} + +fn parse_watermark_digits(raw: &[u8]) -> Result { + let malformed = + || ProtocolError::InvalidReply("watermark is not a nonnegative safe integer".to_string()); + if raw.is_empty() || !raw.iter().all(u8::is_ascii_digit) { + return Err(malformed()); + } + let mut value: u64 = 0; + for digit in raw.iter().skip_while(|byte| **byte == b'0') { + value = value + .checked_mul(10) + .and_then(|scaled| scaled.checked_add(u64::from(digit - b'0'))) + .ok_or_else(malformed)?; + } + if value > MAX_SAFE_INTEGER { + return Err(malformed()); + } + Ok(value) } /// Classify one raw read with the protocol's precedence: absent value, /// unsupported frame, tracked marker validity and zero stamp, fence, /// payload encoding, then a hit. -pub fn decode_frame(raw: Option<&[u8]>, tracked: bool, watermark: Option<&[u8]>) -> Result { - let _ = (raw, tracked, watermark); - todo!("decode_frame") +/// +/// `watermark` is consulted only when `tracked`. A miss carries the parsed +/// fence whenever the marker was valid (including the absent baseline, which +/// carries none); a malformed marker yields an unclassified miss without one. +/// The stamp is returned with full `u64` precision; the safe-integer check +/// belongs to [`normalize_read_result`]. +pub fn decode_frame( + raw: Option<&[u8]>, + tracked: bool, + watermark: Option<&[u8]>, +) -> Result { + let (fence, valid_marker) = if tracked { + match parse_watermark(watermark) { + None => (None, true), + Some(Ok(value)) => (Some(value), true), + Some(Err(_)) => (None, false), + } + } else { + (None, true) + }; + let miss = |reason: MissReason| { + Ok(ReadResult::Miss { + reason, + observed_watermark_ms: fence, + }) + }; + + let Some(raw) = raw else { + return miss(MissReason::ValueAbsent); + }; + if raw.len() < FRAME_HEADER_LEN || raw[0] != FRAME_VERSION { + return miss(MissReason::Unclassified); + } + let stamp = u64::from_be_bytes( + raw[1..9] + .try_into() + .expect("frame header holds eight stamp bytes"), + ); + if tracked && (!valid_marker || stamp == 0) { + return miss(MissReason::Unclassified); + } + if let (true, Some(fence)) = (tracked, fence) { + if stamp <= fence { + return miss(MissReason::WatermarkFenced); + } + } + let payload = match raw[9] { + ENCODING_TEXT => Payload { + bytes: replacement_utf8(&raw[FRAME_HEADER_LEN..]), + binary: false, + }, + ENCODING_BINARY => Payload::binary(&raw[FRAME_HEADER_LEN..]), + _ => return Err(ProtocolError::PayloadEncoding), + }; + Ok(ReadResult::Hit(Frame { + created_at_ms: stamp, + payload, + })) } /// Interpret an untrusted JSON-shaped semantic reply from a custom adapter. /// Unknown shapes become unclassified misses; frame-shaped objects ignore /// stray miss metadata. +/// +/// Only `"kind": "miss"` selects the miss branch. Its reason is parsed from +/// `"reason"` (unknown strings become unclassified) and its +/// `observedWatermarkMs` is kept only when it is a JSON number in the +/// timestamp domain. Any other object is frame-shaped: `createdAtMs` must be a +/// number in the timestamp domain, and a JSON string `payload` is text while +/// anything else is an empty text payload. The result still needs +/// [`normalize_read_result`] for the tracked-key fence rules. pub fn read_result_from_untrusted_json(value: &serde_json::Value) -> ReadResult { - let _ = value; - todo!("read_result_from_untrusted_json") + let Some(object) = value.as_object() else { + return ReadResult::miss(MissReason::Unclassified); + }; + if object.get("kind").and_then(serde_json::Value::as_str) == Some("miss") { + let reason = object + .get("reason") + .and_then(serde_json::Value::as_str) + .and_then(MissReason::parse) + .unwrap_or(MissReason::Unclassified); + let observed_watermark_ms = object + .get("observedWatermarkMs") + .and_then(serde_json::Value::as_f64) + .and_then(|number| validate_timestamp_ms(number).ok()); + return ReadResult::Miss { + reason, + observed_watermark_ms, + }; + } + let Some(created_at_ms) = object + .get("createdAtMs") + .and_then(serde_json::Value::as_f64) + .and_then(|number| validate_timestamp_ms(number).ok()) + else { + return ReadResult::miss(MissReason::Unclassified); + }; + let payload = match object.get("payload").and_then(serde_json::Value::as_str) { + Some(text) => Payload::text(text), + None => Payload::default(), + }; + ReadResult::Hit(Frame { + created_at_ms, + payload, + }) } /// The core trust boundary above wire decoding: drop fences on untracked /// keys or outside the safe domain, and demote a fenced claim without a fence. pub fn normalize_read_result(result: ReadResult, tracked: bool) -> ReadResult { - let _ = (result, tracked); - todo!("normalize_read_result") + match result { + ReadResult::Miss { + reason, + observed_watermark_ms, + } => { + let observed_watermark_ms = + observed_watermark_ms.filter(|fence| tracked && *fence <= MAX_SAFE_INTEGER); + let reason = match reason { + MissReason::WatermarkFenced if observed_watermark_ms.is_none() => { + MissReason::Unclassified + } + other => other, + }; + ReadResult::Miss { + reason, + observed_watermark_ms, + } + } + ReadResult::Hit(frame) if frame.created_at_ms > MAX_SAFE_INTEGER => { + ReadResult::miss(MissReason::Unclassified) + } + hit @ ReadResult::Hit(_) => hit, + } } /// A nonnegative safe-integer timestamp. pub fn validate_timestamp_ms(value: f64) -> Result { - let _ = value; - todo!("validate_timestamp_ms") + if !value.is_finite() + || value < 0.0 + || value.trunc() != value + || value > MAX_SAFE_INTEGER as f64 + { + return Err(ProtocolError::InvalidTimestamp); + } + Ok(value as u64) } /// Round a write TTL up to whole milliseconds and check the 365-day domain. pub fn ceil_supported_cache_ttl_ms(value: f64) -> Result { - let _ = value; - todo!("ceil_supported_cache_ttl_ms") + let ceiled = value.ceil(); + if !ceiled.is_finite() || ceiled <= 0.0 || ceiled > MAX_SUPPORTED_DURATION_MS as f64 { + return Err(ProtocolError::InvalidDuration); + } + Ok(ceiled as u64) +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + fn text_frame(stamp: u64, text: &str) -> Vec { + encode_frame(&Frame { + created_at_ms: stamp, + payload: Payload::text(text), + }) + .expect("encodes") + } + + #[test] + fn encode_layout_and_domain() { + assert_eq!(text_frame(1, "A"), [1, 0, 0, 0, 0, 0, 0, 0, 1, 0, b'A']); + let binary = Frame { + created_at_ms: 0x0102, + payload: Payload::binary(vec![0xFF]), + }; + assert_eq!( + encode_frame(&binary).unwrap(), + [1, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0xFF] + ); + let unsafe_stamp = Frame { + created_at_ms: MAX_SAFE_INTEGER + 1, + payload: Payload::text(""), + }; + assert_eq!( + encode_frame(&unsafe_stamp), + Err(ProtocolError::InvalidTimestamp) + ); + // Ill-formed text is replaced at encode time; binary bytes stay exact. + let ill_formed = Frame { + created_at_ms: 1, + payload: Payload { + bytes: vec![0xE2, 0x82], + binary: false, + }, + }; + assert_eq!( + &encode_frame(&ill_formed).unwrap()[10..], + [0xEF, 0xBF, 0xBD] + ); + } + + #[test] + fn watermark_parsing() { + assert_eq!(parse_watermark(None), None); + assert_eq!(parse_watermark(Some(b"0")), Some(Ok(0))); + assert_eq!( + parse_watermark(Some(b"000000000000000000000000000042")), + Some(Ok(42)) + ); + assert_eq!( + parse_watermark(Some(b"9007199254740991")), + Some(Ok(MAX_SAFE_INTEGER)) + ); + for malformed in [ + &b""[..], + b"9007199254740992", + b"99999999999999999999999", + b"-1", + b"1.0", + b" 1", + b"1a", + "١".as_bytes(), + ] { + assert!( + matches!( + parse_watermark(Some(malformed)), + Some(Err(ProtocolError::InvalidReply(_))) + ), + "{malformed:?}" + ); + } + } + + #[test] + fn decode_precedence() { + let frame = text_frame(1000, "cached"); + let fenced = decode_frame(Some(&frame), true, Some(b"1000")).unwrap(); + assert_eq!( + fenced, + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(1000) + } + ); + let hit = decode_frame(Some(&frame), true, Some(b"999")).unwrap(); + assert_eq!( + hit, + ReadResult::Hit(Frame { + created_at_ms: 1000, + payload: Payload::text("cached") + }) + ); + // Untracked reads never consult the marker. + assert_eq!( + decode_frame(Some(&frame), false, Some(b"garbage")).unwrap(), + hit + ); + // Absent value carries a valid fence; a malformed marker drops it. + assert_eq!( + decode_frame(None, true, Some(b"7")).unwrap(), + ReadResult::Miss { + reason: MissReason::ValueAbsent, + observed_watermark_ms: Some(7) + } + ); + assert_eq!( + decode_frame(None, true, Some(b"x")).unwrap(), + ReadResult::miss(MissReason::ValueAbsent) + ); + // Short or unversioned frames are unclassified before the marker is examined. + assert_eq!( + decode_frame(Some(&frame[..9]), true, Some(b"5")).unwrap(), + ReadResult::Miss { + reason: MissReason::Unclassified, + observed_watermark_ms: Some(5) + } + ); + assert_eq!( + decode_frame(Some(b""), false, None).unwrap(), + ReadResult::miss(MissReason::Unclassified) + ); + // Zero stamps and malformed markers are unclassified only on tracked keys. + let zero = text_frame(0, "z"); + assert_eq!( + decode_frame(Some(&zero), true, None).unwrap(), + ReadResult::miss(MissReason::Unclassified) + ); + assert!(matches!( + decode_frame(Some(&zero), false, None).unwrap(), + ReadResult::Hit(_) + )); + assert_eq!( + decode_frame(Some(&frame), true, Some(b"")).unwrap(), + ReadResult::miss(MissReason::Unclassified) + ); + // The fence check precedes the encoding tag; a fenced bad tag is a miss. + let mut bad_tag = frame.clone(); + bad_tag[9] = 2; + assert_eq!( + decode_frame(Some(&bad_tag), false, None), + Err(ProtocolError::PayloadEncoding) + ); + assert_eq!( + decode_frame(Some(&bad_tag), true, Some(b"1000")).unwrap(), + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(1000) + } + ); + // Full u64 stamps survive decoding; normalization demotes them. + let mut huge = text_frame(1, "h"); + huge[1..9].copy_from_slice(&u64::MAX.to_be_bytes()); + let decoded = decode_frame(Some(&huge), false, None).unwrap(); + assert!(matches!( + decoded, + ReadResult::Hit(Frame { + created_at_ms: u64::MAX, + .. + }) + )); + assert_eq!( + normalize_read_result(decoded, false), + ReadResult::miss(MissReason::Unclassified) + ); + } + + #[test] + fn untrusted_json_shapes() { + assert_eq!( + read_result_from_untrusted_json(&json!(null)), + ReadResult::miss(MissReason::Unclassified) + ); + assert_eq!( + read_result_from_untrusted_json(&json!([1])), + ReadResult::miss(MissReason::Unclassified) + ); + assert_eq!( + read_result_from_untrusted_json( + &json!({"kind": "miss", "reason": "watermark_fenced", "observedWatermarkMs": 5}) + ), + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(5) + } + ); + assert_eq!( + read_result_from_untrusted_json( + &json!({"kind": "miss", "reason": "bogus", "observedWatermarkMs": "5"}) + ), + ReadResult::miss(MissReason::Unclassified) + ); + assert_eq!( + read_result_from_untrusted_json( + &json!({"kind": "miss", "reason": "expired", "observedWatermarkMs": 1.5}) + ), + ReadResult::miss(MissReason::Expired) + ); + // Frame-shaped objects ignore stray miss metadata and non-string payloads. + assert_eq!( + read_result_from_untrusted_json( + &json!({"kind": "hit", "createdAtMs": 3, "payload": "p", "reason": "expired"}) + ), + ReadResult::Hit(Frame { + created_at_ms: 3, + payload: Payload::text("p") + }) + ); + assert_eq!( + read_result_from_untrusted_json(&json!({"createdAtMs": 3, "payload": [1, 2]})), + ReadResult::Hit(Frame { + created_at_ms: 3, + payload: Payload::default() + }) + ); + for bad in [ + json!({"createdAtMs": -1}), + json!({"createdAtMs": "3"}), + json!({"payload": "p"}), + json!({"createdAtMs": 1.5}), + ] { + assert_eq!( + read_result_from_untrusted_json(&bad), + ReadResult::miss(MissReason::Unclassified), + "{bad}" + ); + } + } + + #[test] + fn normalization_rules() { + let fenced = ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(9), + }; + assert_eq!(normalize_read_result(fenced.clone(), true), fenced); + assert_eq!( + normalize_read_result(fenced, false), + ReadResult::miss(MissReason::Unclassified) + ); + let unsafe_fence = ReadResult::Miss { + reason: MissReason::Expired, + observed_watermark_ms: Some(MAX_SAFE_INTEGER + 1), + }; + assert_eq!( + normalize_read_result(unsafe_fence, true), + ReadResult::miss(MissReason::Expired) + ); + let hit = ReadResult::Hit(Frame { + created_at_ms: MAX_SAFE_INTEGER, + payload: Payload::text("ok"), + }); + assert_eq!(normalize_read_result(hit.clone(), true), hit); + } + + #[test] + fn timestamp_and_duration_domains() { + assert_eq!(validate_timestamp_ms(0.0), Ok(0)); + assert_eq!( + validate_timestamp_ms(MAX_SAFE_INTEGER as f64), + Ok(MAX_SAFE_INTEGER) + ); + for bad in [ + -1.0, + 1.5, + 9007199254740992.0, + f64::NAN, + f64::INFINITY, + f64::NEG_INFINITY, + -0.5, + ] { + assert_eq!( + validate_timestamp_ms(bad), + Err(ProtocolError::InvalidTimestamp), + "{bad}" + ); + } + assert_eq!(validate_timestamp_ms(-0.0), Ok(0)); + assert_eq!(ceil_supported_cache_ttl_ms(0.1), Ok(1)); + assert_eq!(ceil_supported_cache_ttl_ms(1.1), Ok(2)); + assert_eq!( + ceil_supported_cache_ttl_ms(31535999999.999), + Ok(MAX_SUPPORTED_DURATION_MS) + ); + assert_eq!( + ceil_supported_cache_ttl_ms(MAX_SUPPORTED_DURATION_MS as f64), + Ok(MAX_SUPPORTED_DURATION_MS) + ); + for bad in [ + 0.0, + -1.0, + 31536000000.001, + f64::NAN, + f64::INFINITY, + f64::NEG_INFINITY, + ] { + assert_eq!( + ceil_supported_cache_ttl_ms(bad), + Err(ProtocolError::InvalidDuration), + "{bad}" + ); + } + } } diff --git a/rust/src/protocol/text.rs b/rust/src/protocol/text.rs index f049f1ef..905f523a 100644 --- a/rust/src/protocol/text.rs +++ b/rust/src/protocol/text.rs @@ -1,8 +1,130 @@ //! Text payload conversion. +//! +//! Text frame payloads and decompressed `0x01` envelopes may hold any bytes. +//! `formal/PROTOCOL.md` ("Text payload domain") fixes how they become text: +//! the WHATWG UTF-8 decoder with replacement error handling and without BOM +//! removal, so every maximal ill-formed subpart becomes exactly one U+FFFD. + +/// The UTF-8 encoding of U+FFFD. +const REPLACEMENT: [u8; 3] = [0xEF, 0xBF, 0xBD]; /// Decode bytes as UTF-8 with one U+FFFD per maximal ill-formed subpart and /// no BOM removal (the WHATWG decoder with replacement error handling). +/// +/// The result is always valid UTF-8. Well-formed input is returned unchanged, +/// so valid text costs one validation pass and one copy. pub fn replacement_utf8(bytes: &[u8]) -> Vec { - let _ = bytes; - todo!("replacement_utf8") + if std::str::from_utf8(bytes).is_ok() { + return bytes.to_vec(); + } + let mut out = Vec::with_capacity(bytes.len() + REPLACEMENT.len()); + let mut rest = bytes; + while !rest.is_empty() { + match std::str::from_utf8(rest) { + Ok(_) => { + out.extend_from_slice(rest); + break; + } + Err(error) => { + let valid = error.valid_up_to(); + out.extend_from_slice(&rest[..valid]); + rest = &rest[valid..]; + let consumed = ill_formed_subpart_len(rest); + out.extend_from_slice(&REPLACEMENT); + rest = &rest[consumed..]; + } + } + } + out +} + +/// Length of the maximal ill-formed subpart at the start of `bytes`, whose +/// first byte is known not to begin a well-formed sequence. +/// +/// A lead in `C2..DF`, `E0..EF` or `F0..F4` swallows the continuation bytes +/// that are valid *in their position* (including the narrowed second-byte +/// ranges after `E0`, `ED`, `F0` and `F4`); the first byte that is not valid +/// there is left to be reprocessed as the next lead. Any other first byte is a +/// one-byte subpart. +fn ill_formed_subpart_len(bytes: &[u8]) -> usize { + let lead = bytes[0]; + let wanted = match lead { + 0xC2..=0xDF => 2, + 0xE0..=0xEF => 3, + 0xF0..=0xF4 => 4, + _ => return 1, + }; + let mut consumed = 1; + while consumed < wanted && consumed < bytes.len() { + let byte = bytes[consumed]; + let (lower, upper) = if consumed == 1 { + match lead { + 0xE0 => (0xA0, 0xBF), + 0xED => (0x80, 0x9F), + 0xF0 => (0x90, 0xBF), + 0xF4 => (0x80, 0x8F), + _ => (0x80, 0xBF), + } + } else { + (0x80, 0xBF) + }; + if byte < lower || byte > upper { + break; + } + consumed += 1; + } + consumed +} + +#[cfg(test)] +mod tests { + use super::replacement_utf8; + + fn decoded(bytes: &[u8]) -> String { + String::from_utf8(replacement_utf8(bytes)).expect("replacement output is UTF-8") + } + + #[test] + fn protocol_replacement_table() { + // The representative outcomes fixed by formal/PROTOCOL.md. + assert_eq!(decoded(&[0x22, 0xFF, 0x22]), "\"\u{FFFD}\""); + assert_eq!(decoded(&[0xE2, 0x82]), "\u{FFFD}"); + assert_eq!(decoded(&[0xE2, 0x82, 0x41]), "\u{FFFD}A"); + assert_eq!(decoded(&[0xED, 0xA0, 0x80]), "\u{FFFD}\u{FFFD}\u{FFFD}"); + assert_eq!( + decoded(&[0xF4, 0x90, 0x80, 0x80]), + "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}" + ); + assert_eq!(decoded(&[0xEF, 0xBB, 0xBF, 0x61]), "\u{FEFF}a"); + } + + #[test] + fn well_formed_input_is_unchanged() { + let text = "\u{80}\u{7FF}\u{800}\u{D7FF}\u{E000}\u{FFFF}\u{10000}\u{10FFFF}\0"; + assert_eq!(replacement_utf8(text.as_bytes()), text.as_bytes()); + assert!(replacement_utf8(b"").is_empty()); + } + + #[test] + fn isolated_continuations_and_overlong_forms_replace_separately() { + assert_eq!(decoded(&[0x80, 0xBF]), "\u{FFFD}\u{FFFD}"); + assert_eq!( + decoded(&[0xC0, 0xAF, 0xE0, 0x80, 0x80]), + "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}" + ); + // A lead that reaches end of input after accepted continuations. + assert_eq!(decoded(&[0xF0, 0x9F, 0x98]), "\u{FFFD}"); + // The narrowed second byte after F0 rejects 8F; 8F is then a lone continuation. + assert_eq!( + decoded(&[0xF0, 0x8F, 0x80, 0x80]), + "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}" + ); + // A complete prefix interrupted by a new lead keeps the second sequence intact. + assert_eq!(decoded(&[0xE2, 0x82, 0xC3, 0xA9]), "\u{FFFD}\u{E9}"); + // Leads beyond F4 and FE/FF are single-byte subparts. + assert_eq!( + decoded(&[0xF5, 0x80, 0xFE, 0xFF, 0x41]), + "\u{FFFD}\u{FFFD}\u{FFFD}\u{FFFD}A" + ); + } } diff --git a/rust/tests/formal/fixtures_tmp.rs b/rust/tests/formal/fixtures_tmp.rs new file mode 100644 index 00000000..d99279a0 --- /dev/null +++ b/rust/tests/formal/fixtures_tmp.rs @@ -0,0 +1,169 @@ +//! TEMPORARY stand-in for `tests/formal/fixtures.rs`, which another agent +//! writes concurrently with the same function names. The integrator switches +//! `#[path = "formal/fixtures_tmp.rs"]` in `tests/protocol_frames.rs` to +//! `formal/fixtures.rs` and deletes this file. This copy performs no +//! provenance (source hash) checks. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +use serde_json::Value; + +/// Replaces every lone UTF-16 surrogate escape in a fixture file so the JSON +/// still parses; vector runners substitute the binding's input conversion. +pub const LONE_SURROGATE_MARKER: char = '\u{E000}'; + +/// A path relative to the repository root. +pub fn repo_path(relative: &str) -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(relative) +} + +/// Parse a JSON file whose strings may contain lone UTF-16 surrogate escapes, +/// which `serde_json` rejects; each becomes [`LONE_SURROGATE_MARKER`]. +pub fn load_json_marking_lone_surrogates(path: impl AsRef) -> Value { + let path = path.as_ref(); + let raw = std::fs::read_to_string(path) + .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); + serde_json::from_str(&mark_lone_surrogates(&raw)) + .unwrap_or_else(|error| panic!("parse {}: {error}", path.display())) +} + +fn mark_lone_surrogates(raw: &str) -> String { + let bytes = raw.as_bytes(); + let mut out = String::with_capacity(raw.len()); + let mut index = 0; + let escape_at = |at: usize| -> Option { + let hex = bytes.get(at..at + 6)?; + if &hex[..2] != b"\\u" { + return None; + } + u32::from_str_radix(std::str::from_utf8(&hex[2..]).ok()?, 16).ok() + }; + while index < bytes.len() { + if bytes[index] != b'\\' { + let char_len = utf8_char_len(bytes[index]); + out.push_str(&raw[index..index + char_len]); + index += char_len; + continue; + } + match escape_at(index) { + Some(unit) if (0xD800..=0xDBFF).contains(&unit) => { + if escape_at(index + 6).is_some_and(|low| (0xDC00..=0xDFFF).contains(&low)) { + out.push_str(&raw[index..index + 12]); + index += 12; + } else { + out.push_str(&format!("\\u{:04X}", LONE_SURROGATE_MARKER as u32)); + index += 6; + } + } + Some(unit) if (0xDC00..=0xDFFF).contains(&unit) => { + out.push_str(&format!("\\u{:04X}", LONE_SURROGATE_MARKER as u32)); + index += 6; + } + Some(_) => { + out.push_str(&raw[index..index + 6]); + index += 6; + } + None => { + // Any other escape is two bytes: copy both so `\\` never hides a `\u`. + let end = (index + 2).min(bytes.len()); + out.push_str(&raw[index..end]); + index = end; + } + } + } + out +} + +fn utf8_char_len(lead: u8) -> usize { + match lead { + 0x00..=0x7F => 1, + 0xC0..=0xDF => 2, + 0xE0..=0xEF => 3, + _ => 4, + } +} + +/// The portable protocol vector groups: `formal/protocol-vectors.json` merged +/// with every Quint-generated protocol artifact listed in +/// `formal/execution.json`. `selection` is `"all"`, `"generated"` or +/// `"fixed"` (the values of `DIALCACHE_PROTOCOL_CORPUS`). +pub fn protocol_groups(selection: &str) -> BTreeMap> { + assert!( + matches!(selection, "all" | "generated" | "fixed"), + "unknown protocol corpus selection {selection}" + ); + let fixed = load_json_marking_lone_surrogates(repo_path("formal/protocol-vectors.json")); + assert_eq!( + fixed.get("schemaVersion").and_then(Value::as_u64), + Some(3), + "unsupported protocol vector schema" + ); + let mut groups: BTreeMap> = BTreeMap::new(); + for (name, rows) in fixed.as_object().expect("protocol vectors are an object") { + if name == "schemaVersion" { + continue; + } + let rows = rows + .as_array() + .unwrap_or_else(|| panic!("protocol group {name} is not an array")); + let entry = groups.entry(name.clone()).or_default(); + if selection != "generated" { + entry.extend(rows.iter().cloned()); + } + } + if selection == "fixed" { + return groups; + } + let manifest = load_json_marking_lone_surrogates(repo_path("formal/execution.json")); + let models = manifest + .get("models") + .and_then(Value::as_array) + .expect("execution.json lists models"); + for model in models { + let Some(export) = model.get("vectorExport") else { + continue; + }; + if export.get("kind").and_then(Value::as_str) != Some("protocol") { + continue; + } + let artifact = export + .get("artifact") + .and_then(Value::as_str) + .expect("protocol export names an artifact"); + let generated = load_json_marking_lone_surrogates(repo_path(artifact)); + assert_eq!( + generated.get("schemaVersion").and_then(Value::as_u64), + Some(3), + "{artifact}: unsupported schema" + ); + let mut count = 0; + for (name, rows) in generated + .as_object() + .expect("generated artifact is an object") + { + if name == "schemaVersion" || name == "provenance" { + continue; + } + let rows = rows + .as_array() + .unwrap_or_else(|| panic!("{artifact}: group {name} is not an array")); + let entry = groups + .get_mut(name) + .unwrap_or_else(|| panic!("{artifact}: unknown generated protocol group {name}")); + entry.extend(rows.iter().cloned()); + count += rows.len(); + } + let expected = export + .get("cases") + .and_then(Value::as_u64) + .expect("protocol export declares cases") as usize; + assert_eq!( + count, expected, + "{artifact}: incomplete generated protocol vector inventory" + ); + } + groups +} diff --git a/rust/tests/formal/frame_vectors.rs b/rust/tests/formal/frame_vectors.rs new file mode 100644 index 00000000..a07baa17 --- /dev/null +++ b/rust/tests/formal/frame_vectors.rs @@ -0,0 +1,388 @@ +//! Runners for the frame, decode, timestamp, duration and envelope vector +//! groups of the portable protocol corpus. Each mirrors the corresponding +//! part of `go/protocol_test.go` field for field. + +use dialcache::limits::MAX_DECOMPRESSED_BYTES; +use dialcache::observe::CompressionOutcome; +use dialcache::protocol::{ + ceil_supported_cache_ttl_ms, compress_payload, decode_frame, decompress_payload, encode_frame, + escape_raw_payload, validate_timestamp_ms, CompressionConfig, ProtocolError, + MARKER_ZSTD_BINARY, MARKER_ZSTD_UTF8, +}; +use dialcache::{Frame, Payload, ReadResult}; +use serde_json::{json, Value}; + +use super::fixtures::LONE_SURROGATE_MARKER; + +/// The fixture level every `codecBytes` entry was measured at. +const FIXTURE_ZSTD_LEVEL: i32 = 3; + +fn field<'a>(vector: &'a Value, name: &str) -> Result<&'a Value, String> { + vector + .get(name) + .ok_or_else(|| format!("missing field {name}")) +} + +fn text_field<'a>(vector: &'a Value, name: &str) -> Result<&'a str, String> { + field(vector, name)? + .as_str() + .ok_or_else(|| format!("field {name} is not a string")) +} + +fn unhex(text: &str) -> Result, String> { + hex::decode(text).map_err(|error| format!("invalid hex {text:?}: {error}")) +} + +fn usize_field(vector: &Value, name: &str) -> Result { + let number = field(vector, name)? + .as_u64() + .ok_or_else(|| format!("field {name} is not an unsigned integer"))?; + usize::try_from(number).map_err(|_| format!("field {name} exceeds usize")) +} + +/// The `payloadType` / `payloadUtf8` / `payloadHex` triple as a payload. +/// `replace_lone_surrogates` applies the binding's input conversion: an +/// unpaired UTF-16 surrogate in caller text becomes U+FFFD. +fn vector_payload(vector: &Value, replace_lone_surrogates: bool) -> Result { + match text_field(vector, "payloadType")? { + "binary" => Ok(Payload::binary(unhex(text_field(vector, "payloadHex")?)?)), + "string" => { + let text = text_field(vector, "payloadUtf8")?; + if replace_lone_surrogates { + Ok(Payload::text( + text.replace(LONE_SURROGATE_MARKER, "\u{FFFD}"), + )) + } else { + Ok(Payload::text(text)) + } + } + other => Err(format!("unknown payloadType {other:?}")), + } +} + +/// A numeric fixture input; JSON carries nonfinite values as `specialInput`. +fn number_input(vector: &Value) -> Result { + match vector.get("specialInput").and_then(Value::as_str) { + Some("NaN") => Ok(f64::NAN), + Some("Infinity") => Ok(f64::INFINITY), + Some("-Infinity") => Ok(f64::NEG_INFINITY), + Some(other) => Err(format!("unknown specialInput {other:?}")), + None => field(vector, "input")? + .as_f64() + .ok_or_else(|| "input is not a number".to_string()), + } +} + +fn outcome_name(outcome: Option) -> &'static str { + outcome.map_or("passthrough", CompressionOutcome::as_str) +} + +fn max_decompressed_bytes(vector: &Value) -> Result { + match vector.get("maxDecompressedBytes") { + None | Some(Value::Null) => Ok(MAX_DECOMPRESSED_BYTES), + Some(_) => usize_field(vector, "maxDecompressedBytes"), + } +} + +/// `frameVectors`: encoding a payload with a stamp yields the exact frame bytes. +pub fn check_frame_vector(vector: &Value) -> Result<(), String> { + let created_at_ms = field(vector, "createdAtMs")? + .as_u64() + .ok_or("createdAtMs is not an unsigned integer")?; + let payload = vector_payload(vector, true)?; + let frame = encode_frame(&Frame { + created_at_ms, + payload, + }) + .map_err(|error| format!("encode failed: {error}"))?; + let actual = hex::encode(&frame); + let expected = text_field(vector, "frameHex")?; + if actual != expected { + return Err(format!("frame: got {actual}, want {expected}")); + } + Ok(()) +} + +pub fn check_tracked_decode_vector(vector: &Value) -> Result<(), String> { + check_decode_vector(vector, true) +} + +pub fn check_untracked_decode_vector(vector: &Value) -> Result<(), String> { + check_decode_vector(vector, false) +} + +/// `trackedDecodeVectors` / `untrackedDecodeVectors`: `frameHex` null is an +/// absent value and `""` a present empty one; `watermarkUtf8` null is absent. +/// The classified result is compared with `expected` as JSON, retaining +/// integer precision. +fn check_decode_vector(vector: &Value, tracked: bool) -> Result<(), String> { + let raw = match field(vector, "frameHex")? { + Value::Null => None, + Value::String(text) => Some(unhex(text)?), + _ => return Err("frameHex is neither null nor a string".to_string()), + }; + let watermark = match vector.get("watermarkUtf8").unwrap_or(&Value::Null) { + Value::Null => None, + Value::String(text) => Some(text.as_bytes().to_vec()), + _ => return Err("watermarkUtf8 is neither null nor a string".to_string()), + }; + let actual = match decode_frame(raw.as_deref(), tracked, watermark.as_deref()) { + Err(ProtocolError::PayloadEncoding) => json!({ "kind": "payload_encoding_error" }), + Err(other) => return Err(format!("unexpected decode error {other}")), + Ok(ReadResult::Miss { + reason, + observed_watermark_ms, + }) => { + let mut object = json!({ "kind": "miss", "reason": reason.as_str() }); + if let Some(fence) = observed_watermark_ms { + object["observedWatermarkMs"] = json!(fence); + } + object + } + Ok(ReadResult::Hit(frame)) => { + let mut object = json!({ "kind": "hit", "createdAtMs": frame.created_at_ms }); + if frame.payload.binary { + object["payloadType"] = json!("binary"); + object["payloadHex"] = json!(hex::encode(&frame.payload.bytes)); + } else { + object["payloadType"] = json!("string"); + let text = String::from_utf8(frame.payload.bytes) + .map_err(|_| "text payload is not UTF-8")?; + object["payloadUtf8"] = json!(text); + } + object + } + }; + let expected = field(vector, "expected")?; + if &actual != expected { + return Err(format!("got {actual}\nwant {expected}")); + } + Ok(()) +} + +/// `invalidTimestampVectors`: every input is outside the timestamp domain. +pub fn check_invalid_timestamp_vector(vector: &Value) -> Result<(), String> { + match validate_timestamp_ms(number_input(vector)?) { + Ok(accepted) => Err(format!("invalid timestamp accepted as {accepted}")), + Err(_) => Ok(()), + } +} + +/// `durationVectors`: `expected` null rejects; otherwise the ceiled TTL. +pub fn check_duration_vector(vector: &Value) -> Result<(), String> { + let result = ceil_supported_cache_ttl_ms(number_input(vector)?); + match (field(vector, "expected")?.as_u64(), result) { + (None, Err(_)) => Ok(()), + (None, Ok(accepted)) => Err(format!("invalid duration accepted as {accepted}")), + (Some(expected), Ok(actual)) if actual == expected => Ok(()), + (Some(expected), other) => Err(format!("got {other:?} want {expected}")), + } +} + +/// `envelopeVectors`: escaping, unconditional read interpretation and the +/// escape round trip over raw binary input. +pub fn check_envelope_vector(vector: &Value) -> Result<(), String> { + let raw = Payload::binary(unhex(text_field(vector, "inputHex")?)?); + let escaped = escape_raw_payload(raw.clone()); + if escaped.bytes != unhex(text_field(vector, "escapedHex")?)? { + return Err(format!("escape differs: {}", hex::encode(&escaped.bytes))); + } + let decoded = decompress_payload(raw.clone(), MAX_DECOMPRESSED_BYTES); + let expected_outcome = text_field(vector, "outcome")?; + if outcome_name(decoded.outcome) != expected_outcome + || !decoded.payload.binary + || decoded.payload.bytes != unhex(text_field(vector, "decodedHex")?)? + { + return Err(format!( + "decoded {decoded:?}, want {expected_outcome} {}", + text_field(vector, "decodedHex")? + )); + } + if decompress_payload(escaped, MAX_DECOMPRESSED_BYTES) + .payload + .bytes + != raw.bytes + { + return Err("escape roundtrip differs".to_string()); + } + Ok(()) +} + +/// `compressedDecodeVectors`: the native decoder confirms the environmental +/// `codecFixture`, then the wrapper's outcome, type and bytes are checked +/// under the optional per-call cap. +pub fn check_compressed_decode_vector(vector: &Value) -> Result<(), String> { + let input = unhex(text_field(vector, "inputHex")?)?; + if let Some(fixture) = vector + .get("codecFixture") + .filter(|fixture| !fixture.is_null()) + { + let succeeds = field(fixture, "succeeds")? + .as_bool() + .ok_or("codecFixture.succeeds is not a boolean")?; + let native = zstd::decode_all(input.get(1..).unwrap_or_default()); + match (succeeds, native) { + (true, Ok(decoded)) if decoded == unhex(text_field(fixture, "decodedHex")?)? => {} + (true, other) => return Err(format!("native codec fixture differs: {other:?}")), + (false, Ok(decoded)) => { + return Err(format!( + "native decoder accepted rejected fixture: {}", + hex::encode(decoded) + )) + } + (false, Err(_)) => {} + } + } + let got = decompress_payload(Payload::binary(input), max_decompressed_bytes(vector)?); + let want = vector_payload(vector, false)?; + let outcome = vector + .get("outcome") + .and_then(Value::as_str) + .unwrap_or("decompressed"); + if outcome_name(got.outcome) != outcome || got.payload != want { + return Err(format!("got {got:?} want {outcome} {want:?}")); + } + Ok(()) +} + +/// `compressionWriteVectors`: when this binding's native level-3 encoder +/// reproduces the TypeScript fixture length, the TypeScript per-binding +/// expectations apply; otherwise only the generic outcome and the selection +/// rule are checked. Every result must round-trip and respect the escape rule. +pub fn check_compression_write_vector(vector: &Value) -> Result<(), String> { + let name = vector + .get("name") + .and_then(Value::as_str) + .unwrap_or(""); + let raw = vector_payload(vector, false)?; + let escaped = escape_raw_payload(raw.clone()); + let threshold_bytes = usize_field(vector, "thresholdBytes")?; + let max = max_decompressed_bytes(vector)?; + + let mut modeled = None; + if let Some(codec_bytes) = vector.get("codecBytes").filter(|codec| !codec.is_null()) { + let fixture = usize_field(codec_bytes, "typescript")?; + let native = native_zstd_len(&raw.bytes)?; + if native == fixture { + modeled = Some(field(field(vector, "expectedByBinding")?, "typescript")?); + println!("compressionWriteVectors {name}: native level-3 length {native} matches the TypeScript fixture; using its expectations"); + } else { + println!("compressionWriteVectors {name}: native level-3 length {native} differs from the TypeScript fixture {fixture}; checking the generic outcome and selection rule only"); + } + } + + let got = compress_payload( + raw.clone(), + &CompressionConfig { + threshold_bytes, + level: FIXTURE_ZSTD_LEVEL, + }, + max, + ) + .map_err(|error| format!("compress failed: {error}"))?; + let expected_outcome = match modeled { + Some(expected) => text_field(expected, "outcome")?, + None => text_field(vector, "outcome")?, + }; + if got.outcome.as_str() != expected_outcome { + return Err(format!( + "got outcome {} want {expected_outcome} ({got:?})", + got.outcome + )); + } + if let Some(expected) = modeled { + let stored = usize_field(expected, "storedBytes")?; + if got.stored_bytes != stored { + return Err(format!("storedBytes {} want {stored}", got.stored_bytes)); + } + if vector.get("originalBytes").is_none() + || vector.get("rawStoredBytes").is_none() + || vector.get("escapedHex").is_none() + { + return Err( + "modeled vector lacks originalBytes, rawStoredBytes or escapedHex".to_string(), + ); + } + } + if let Some(original) = vector.get("originalBytes").filter(|value| !value.is_null()) { + let original = original.as_u64().ok_or("originalBytes is not a number")? as usize; + if got.original_bytes != original { + return Err(format!( + "originalBytes {} want {original}", + got.original_bytes + )); + } + } + if let Some(raw_stored) = vector + .get("rawStoredBytes") + .filter(|value| !value.is_null()) + { + let raw_stored = raw_stored + .as_u64() + .ok_or("rawStoredBytes is not a number")? as usize; + if escaped.len() != raw_stored { + return Err(format!( + "escaped length {} want rawStoredBytes {raw_stored}", + escaped.len() + )); + } + } + if let Some(escaped_hex) = vector.get("escapedHex").and_then(Value::as_str) { + if escaped.bytes != unhex(escaped_hex)? { + return Err(format!( + "escaped bytes {} want {escaped_hex}", + hex::encode(&escaped.bytes) + )); + } + } + + let decoded = decompress_payload(got.payload.clone(), MAX_DECOMPRESSED_BYTES); + if decoded.payload != raw { + return Err(format!("compression changed value: {decoded:?}")); + } + if got.outcome == CompressionOutcome::Compressed { + if !got.payload.binary + || got.stored_bytes >= escaped.len() + || got.payload.len() != got.stored_bytes + { + return Err(format!("compression grew or misreported: {got:?}")); + } + let mut marker = if raw.binary { + MARKER_ZSTD_BINARY + } else { + MARKER_ZSTD_UTF8 + }; + if let Some(expected) = modeled { + marker = u8::try_from( + field(expected, "marker")? + .as_i64() + .ok_or("marker is not an integer")?, + ) + .map_err(|_| "modeled marker is not a byte".to_string())?; + } + if got.payload.bytes.first() != Some(&marker) { + return Err(format!( + "compressed payload marker {:?} want {marker}", + got.payload.bytes.first() + )); + } + } else if got.payload != escaped { + return Err(format!("raw representation differs: {got:?}")); + } + Ok(()) +} + +/// The native single-frame, checksum-free zstd length at the fixture level, +/// computed independently of the crate's wrapper. +fn native_zstd_len(bytes: &[u8]) -> Result { + let describe = |code: usize| format!("native zstd: {}", zstd_safe::get_error_name(code)); + let mut context = zstd_safe::CCtx::create(); + context + .set_parameter(zstd_safe::CParameter::CompressionLevel(FIXTURE_ZSTD_LEVEL)) + .map_err(describe)?; + context + .set_parameter(zstd_safe::CParameter::ChecksumFlag(false)) + .map_err(describe)?; + let mut out = vec![0u8; zstd_safe::compress_bound(bytes.len())]; + context.compress2(&mut out[..], bytes).map_err(describe) +} diff --git a/rust/tests/protocol_frames.rs b/rust/tests/protocol_frames.rs new file mode 100644 index 00000000..c88777da --- /dev/null +++ b/rust/tests/protocol_frames.rs @@ -0,0 +1,85 @@ +//! Frame, decode, timestamp, duration and envelope conformance against the +//! portable protocol corpus (`formal/protocol-vectors.json` plus the +//! Quint-generated frame and envelope artifacts). + +// TODO(integrator): switch to `formal/fixtures.rs` and delete `formal/fixtures_tmp.rs`. +#[path = "formal/fixtures_tmp.rs"] +mod fixtures; +#[path = "formal/frame_vectors.rs"] +mod frame_vectors; + +use serde_json::Value; + +type Check = fn(&Value) -> Result<(), String>; + +/// The eight groups this test owns, with their fixed-corpus sizes. +const GROUPS: [(&str, usize, Check); 8] = [ + ("frameVectors", 8, frame_vectors::check_frame_vector), + ( + "trackedDecodeVectors", + 38, + frame_vectors::check_tracked_decode_vector, + ), + ( + "untrackedDecodeVectors", + 18, + frame_vectors::check_untracked_decode_vector, + ), + ( + "invalidTimestampVectors", + 3, + frame_vectors::check_invalid_timestamp_vector, + ), + ("durationVectors", 7, frame_vectors::check_duration_vector), + ("envelopeVectors", 7, frame_vectors::check_envelope_vector), + ( + "compressedDecodeVectors", + 12, + frame_vectors::check_compressed_decode_vector, + ), + ( + "compressionWriteVectors", + 6, + frame_vectors::check_compression_write_vector, + ), +]; + +#[test] +fn protocol_frame_groups_conform() { + let groups = fixtures::protocol_groups("all"); + let generated = fixtures::protocol_groups("generated"); + let mut failures = Vec::new(); + let mut total = 0; + let mut expected_total = 0; + for (name, fixed_count, check) in GROUPS { + let rows = groups + .get(name) + .unwrap_or_else(|| panic!("protocol corpus lacks group {name}")); + let generated_rows = generated.get(name).map_or(0, Vec::len); + expected_total += fixed_count + generated_rows; + let mut passed = 0; + for vector in rows { + total += 1; + let vector_name = vector + .get("name") + .and_then(Value::as_str) + .unwrap_or(""); + match check(vector) { + Ok(()) => passed += 1, + Err(reason) => failures.push(format!("{name} / {vector_name}: {reason}")), + } + } + println!( + "{name}: {passed}/{} ({fixed_count} fixed + {generated_rows} generated)", + rows.len() + ); + } + assert!( + failures.is_empty(), + "{} protocol vector(s) failed:\n{}", + failures.len(), + failures.join("\n") + ); + assert_eq!(total, expected_total, "review protocol vector coverage"); + println!("frame protocol groups exercised: {total} vectors"); +} From 92a035c29f825ea65bb6d6997d1bdad420084828 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 00:29:47 -0700 Subject: [PATCH 07/31] test(rust): port the shared replay harness infrastructure Add the language-neutral replay plumbing the Rust conformance drivers plug into: a JSON-lines client for the shared Node coordinator with a real-time watchdog and exact envelope checks, a local interpreter for protocol.schema.json, strict JSON and ITF decoding, corpus and inventory-id discovery, the JSONL assertion report and the witness-evidence check with a self-contained SHA-256. harness_infra ports the Go transport, JSON, registry and witness controls. --- rust/tests/formal/inventory.rs | 473 +++++++++++++++ rust/tests/formal/json.rs | 223 +++++++ rust/tests/formal/mod.rs | 16 + rust/tests/formal/report.rs | 168 ++++++ rust/tests/formal/schema.rs | 380 ++++++++++++ rust/tests/formal/transport.rs | 490 +++++++++++++++ rust/tests/formal/witness.rs | 386 ++++++++++++ rust/tests/harness_infra.rs | 1039 ++++++++++++++++++++++++++++++++ 8 files changed, 3175 insertions(+) create mode 100644 rust/tests/formal/inventory.rs create mode 100644 rust/tests/formal/json.rs create mode 100644 rust/tests/formal/mod.rs create mode 100644 rust/tests/formal/report.rs create mode 100644 rust/tests/formal/schema.rs create mode 100644 rust/tests/formal/transport.rs create mode 100644 rust/tests/formal/witness.rs create mode 100644 rust/tests/harness_infra.rs diff --git a/rust/tests/formal/inventory.rs b/rust/tests/formal/inventory.rs new file mode 100644 index 00000000..5e7398ea --- /dev/null +++ b/rust/tests/formal/inventory.rs @@ -0,0 +1,473 @@ +//! Corpus discovery and the language-neutral case inventory. +//! +//! Ports the trace selection of the Go harness (`TestCoreConformance`, +//! `effectsPaths`, `featurePaths`, `featureRegressionPaths`), its registry +//! checks (`validateRegistry`, `validateBehaviorProfileRegistry`) and the id +//! scheme of `formal/conformance.mjs` `conformanceInventory`, so the Rust +//! replay reports the same `sampled/…`, `regression/…`, `scenario/…`, +//! `protocol/…` and `witness/…` ids the completion checker requires. + +use super::json::strict_parse; +use serde_json::Value; +use std::path::{Path, PathBuf}; + +/// Absolute repository root (the parent of the `rust/` crate directory). +pub fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("crate lives in /rust") + .to_path_buf() +} + +/// Resolves a repository-relative path such as `formal/profiles.json`. +pub fn repo_path(relative: &str) -> PathBuf { + repo_root().join(relative) +} + +/// Where one replay corpus comes from. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TraceSource { + /// The committed smoke history under `formal/`. + Smoke, + /// One explicit history file. + File(PathBuf), + /// A generated corpus directory plus its scheduled regressions. + Directory(PathBuf), +} + +impl TraceSource { + fn from_env(file: &str, directory: &str, conflict: &str) -> Result { + let file = std::env::var(file).unwrap_or_default(); + let directory = std::env::var(directory).unwrap_or_default(); + match (file.is_empty(), directory.is_empty()) { + (false, false) => Err(conflict.to_string()), + (false, true) => Ok(TraceSource::File(PathBuf::from(file))), + (true, false) => Ok(TraceSource::Directory(PathBuf::from(directory))), + (true, true) => Ok(TraceSource::Smoke), + } + } + + /// Whether this source is a full generated corpus. + pub fn is_directory(&self) -> bool { + matches!(self, TraceSource::Directory(_)) + } +} + +/// Which protocol vector rows a run replays. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProtocolCorpus { + /// Fixed and generated rows (the default and `all`). + All, + /// Quint-generated rows only. + Generated, + /// Checked-in rows only. + Fixed, +} + +/// The environment-driven selection of what one conformance run replays. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Selection { + /// `DIALCACHE_MBT_TRACE_FILE` / `DIALCACHE_MBT_TRACE_DIR`. + pub core: TraceSource, + /// `DIALCACHE_EFFECTS_TRACE_FILE` / `DIALCACHE_EFFECTS_TRACE_DIR`. + pub effects: TraceSource, + /// `DIALCACHE_FEATURE_TRACE_FILE` / `DIALCACHE_FEATURE_TRACE_DIR`. + pub features: TraceSource, + /// `DIALCACHE_FEATURE_PROFILE`: replay one feature profile only. + pub feature_profile: Option, + /// `DIALCACHE_WITNESS_EVIDENCE_DIR`: evaluated witness evidence files. + pub witness_evidence_dir: Option, + /// `DIALCACHE_PROTOCOL_CORPUS`. + pub protocol_corpus: ProtocolCorpus, + /// `DIALCACHE_RUST_REPORT`: JSONL assertion report path. + pub report: Option, + /// `DIALCACHE_BEHAVIOR_SCENARIO`: substring filter on scenario names. + pub behavior_scenario: Option, +} + +fn optional(name: &str) -> Option { + std::env::var(name).ok().filter(|value| !value.is_empty()) +} + +impl Selection { + /// Reads the selection from the environment. + pub fn from_env() -> Result { + let protocol_corpus = match optional("DIALCACHE_PROTOCOL_CORPUS").as_deref() { + None | Some("all") => ProtocolCorpus::All, + Some("generated") => ProtocolCorpus::Generated, + Some("fixed") => ProtocolCorpus::Fixed, + Some(other) => return Err(format!("unknown protocol corpus selection {other}")), + }; + Ok(Selection { + core: TraceSource::from_env( + "DIALCACHE_MBT_TRACE_FILE", + "DIALCACHE_MBT_TRACE_DIR", + "select either a trace file or directory", + )?, + effects: TraceSource::from_env( + "DIALCACHE_EFFECTS_TRACE_FILE", + "DIALCACHE_EFFECTS_TRACE_DIR", + "select either an effects trace file or directory", + )?, + features: TraceSource::from_env( + "DIALCACHE_FEATURE_TRACE_FILE", + "DIALCACHE_FEATURE_TRACE_DIR", + "select either a feature trace file or directory", + )?, + feature_profile: optional("DIALCACHE_FEATURE_PROFILE"), + witness_evidence_dir: optional("DIALCACHE_WITNESS_EVIDENCE_DIR").map(PathBuf::from), + protocol_corpus, + report: optional("DIALCACHE_RUST_REPORT").map(PathBuf::from), + behavior_scenario: optional("DIALCACHE_BEHAVIOR_SCENARIO"), + }) + } + + /// Core histories: the smoke history, one file, or a directory's `*.itf.json` + /// plus the scheduled core regressions. + pub fn core_paths(&self) -> Result, String> { + match &self.core { + TraceSource::Smoke => Ok(vec![repo_path("formal/conformance-smoke.itf.json")]), + TraceSource::File(file) => Ok(vec![file.clone()]), + TraceSource::Directory(directory) => { + let mut paths = glob_itf(directory)?; + if paths.is_empty() { + return Err("empty trace corpus".to_string()); + } + paths.extend(regression_paths("core", directory)?); + Ok(paths) + } + } + } + + /// Effects histories, including scheduled regressions for a directory corpus. + pub fn effects_paths(&self) -> Result, String> { + let paths = match &self.effects { + TraceSource::Smoke => vec![repo_path("formal/effects-smoke.itf.json")], + TraceSource::File(file) => vec![file.clone()], + TraceSource::Directory(directory) => { + let mut paths = glob_itf(directory)?; + paths.extend(regression_paths("effects", directory)?); + paths + } + }; + if paths.is_empty() { + return Err("empty effects corpus".to_string()); + } + Ok(paths) + } + + /// Histories of one feature profile (including `local-clock`). An explicit + /// file is used only when it belongs to the profile; a directory corpus + /// reads `//*.itf.json` plus the scheduled regressions. + pub fn feature_paths(&self, profile: &str) -> Result, String> { + match &self.features { + TraceSource::File(file) => { + let parent = file + .parent() + .and_then(Path::file_name) + .map(|name| name.to_string_lossy().into_owned()); + let name = file_name(file); + if parent.as_deref() == Some(profile) || name.contains(&format!("{profile}-smoke")) + { + Ok(vec![file.clone()]) + } else { + Ok(Vec::new()) + } + } + TraceSource::Directory(directory) => { + let mut paths = glob_itf(&directory.join(profile))?; + if paths.is_empty() { + return Err(format!("empty feature corpus {profile}")); + } + paths.extend(regression_paths(profile, directory)?); + Ok(paths) + } + TraceSource::Smoke => Ok(vec![repo_path(&format!("formal/{profile}-smoke.itf.json"))]), + } + } +} + +fn file_name(path: &Path) -> String { + path.file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// Sorted `*.itf.json` files directly inside `directory`. +pub fn glob_itf(directory: &Path) -> Result, String> { + let entries = match std::fs::read_dir(directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(format!("{}: {error}", directory.display())), + }; + let mut names: Vec = entries + .filter_map(Result::ok) + .map(|entry| entry.file_name().to_string_lossy().into_owned()) + .filter(|name| name.ends_with(".itf.json")) + .collect(); + names.sort(); + Ok(names.into_iter().map(|name| directory.join(name)).collect()) +} + +/// Lexically resolves `/..` the way Go's `filepath.Join` does. +fn parent_of(directory: &Path) -> PathBuf { + let mut parent = directory.to_path_buf(); + match directory.components().next_back() { + Some(std::path::Component::Normal(_)) => { + parent.pop(); + } + _ => parent.push(".."), + } + parent +} + +/// Scheduled Quint regressions of `profile`, expected at +/// `/../regressions//.itf.json`. +pub fn regression_paths(profile: &str, directory: &Path) -> Result, String> { + let regressions = parent_of(directory).join("regressions").join(profile); + let mut paths = Vec::new(); + for name in replay_regressions(profile)? { + let path = regressions.join(format!("{name}.itf.json")); + if let Err(error) = std::fs::metadata(&path) { + return Err(format!( + "missing Quint regression {name}: {}: {error}", + path.display() + )); + } + paths.push(path); + } + Ok(paths) +} + +/// `replayRegressions` of the `formal/execution.json` model bound to `profile`. +pub fn replay_regressions(profile: &str) -> Result, String> { + let manifest = read_json("formal/execution.json")?; + let mut names = Vec::new(); + for model in manifest + .get("models") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + if model.get("profile").and_then(Value::as_str) != Some(profile) { + continue; + } + for name in model + .get("replayRegressions") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + match name.as_str() { + Some(name) => names.push(name.to_string()), + None => { + return Err( + "malformed replayRegressions entry in formal/execution.json".to_string() + ) + } + } + } + } + Ok(names) +} + +fn read_json(relative: &str) -> Result { + let path = repo_path(relative); + let text = + std::fs::read_to_string(&path).map_err(|error| format!("{}: {error}", path.display()))?; + strict_parse(&text).map_err(|error| format!("{relative}: {error}")) +} + +/// A history's kind follows the corpus layout the shared evaluator classifies +/// by: exported regressions live under `regressions//`, every other +/// replayed history is a sampled one. +pub fn trace_kind(path: &Path) -> &'static str { + let grandparent = path + .parent() + .and_then(Path::parent) + .and_then(Path::file_name); + if grandparent.is_some_and(|name| name == "regressions") { + "regression" + } else { + "sampled" + } +} + +/// Inventory id of one replayed history: `regression//` for an +/// exported regression, `sampled//` for `trace_.itf.json` of a +/// full generated corpus, otherwise `smoke//`. +pub fn trace_case_id(profile: &str, path: &Path, full: bool) -> String { + let name = file_name(path); + let stem = name.strip_suffix(".itf.json").unwrap_or(&name); + if trace_kind(path) == "regression" { + return format!("regression/{profile}/{stem}"); + } + if full { + if let Some(index) = stem.strip_prefix("trace_") { + if !index.is_empty() + && index.bytes().all(|byte| byte.is_ascii_digit()) + && (index == "0" || !index.starts_with('0')) + { + return format!("sampled/{profile}/{index}"); + } + } + } + format!("smoke/{profile}/{name}") +} + +/// `scenario//` with percent-encoded components. +pub fn scenario_case_id(feature: &str, name: &str) -> String { + format!( + "scenario/{}/{}", + percent_encode_component(feature), + percent_encode_component(name) + ) +} + +/// `protocol//` with a percent-encoded vector name. +pub fn protocol_case_id(group: &str, name: &str) -> String { + format!("protocol/{group}/{}", percent_encode_component(name)) +} + +/// `witness/`. +pub fn witness_case_id(profile: &str) -> String { + format!("witness/{profile}") +} + +/// Exactly JavaScript's `encodeURIComponent`: every byte of the UTF-8 encoding +/// outside `A-Z a-z 0-9 - _ . ! ~ * ' ( )` becomes `%XX` with uppercase hex. +pub fn percent_encode_component(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() || b"-_.!~*'()".contains(&byte) { + out.push(byte as char); + } else { + out.push_str(&format!("%{byte:02X}")); + } + } + out +} + +/// Behavior profile versions this port implements, from +/// `go/behavior_registry_test.go`. +pub const BEHAVIOR_PROFILE_VERSIONS: [(&str, i64); 14] = [ + ("recovery-read", 1), + ("local-failure", 1), + ("runtime-boundaries", 1), + ("shadow-layers", 1), + ("local-clock", 1), + ("source-budgets", 1), + ("effects", 2), + ("scope", 2), + ("policy", 3), + ("layers", 2), + ("recovery", 1), + ("independent", 2), + ("shadow", 3), + ("admission", 1), +]; + +/// The version this port implements of a behavior profile, if it knows it. +pub fn behavior_profile_version(profile: &str) -> Option { + BEHAVIOR_PROFILE_VERSIONS + .iter() + .find(|(name, _)| *name == profile) + .map(|(_, version)| *version) +} + +/// Ports `validateRegistry`: `formal/profiles.json` must declare schema 1, +/// specification 0.1.0, protocol schema 3 and exactly one core profile at version 1. +pub fn registry_check() -> Result<(), String> { + registry_check_text(&read_registry_text()?) +} + +fn read_registry_text() -> Result { + let path = repo_path("formal/profiles.json"); + std::fs::read_to_string(&path).map_err(|error| format!("{}: {error}", path.display())) +} + +/// [`registry_check`] over explicit registry text. +pub fn registry_check_text(raw: &str) -> Result<(), String> { + let registry = strict_parse(raw)?; + if registry.get("schemaVersion").and_then(Value::as_f64) != Some(1.0) + || registry.get("specificationVersion").and_then(Value::as_str) != Some("0.1.0") + || registry + .get("protocolSchemaVersion") + .and_then(Value::as_f64) + != Some(3.0) + { + return Err("unsupported specification/profile/protocol registry version".to_string()); + } + let mut core = 0; + for profile in registry + .get("profiles") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + if profile.get("id").and_then(Value::as_str) == Some("core") { + core += 1; + if profile.get("version").and_then(Value::as_f64) != Some(1.0) { + return Err("unsupported core profile version".to_string()); + } + } + } + if core != 1 { + return Err("registry requires exactly one core profile".to_string()); + } + Ok(()) +} + +/// Ports `validateBehaviorProfileRegistry` against `formal/profiles.json`. +pub fn profile_registry_check(profile: &str, version: i64) -> Result<(), String> { + profile_registry_check_text(&read_registry_text()?, profile, version) +} + +/// [`profile_registry_check`] over explicit registry text. +pub fn profile_registry_check_text(raw: &str, name: &str, version: i64) -> Result<(), String> { + let registry = strict_parse(raw)?; + if registry.get("schemaVersion").and_then(Value::as_f64) != Some(1.0) + || registry.get("specificationVersion").and_then(Value::as_str) != Some("0.1.0") + || registry + .get("behavioralSchemaVersion") + .and_then(Value::as_f64) + != Some(2.0) + || registry + .get("protocolSchemaVersion") + .and_then(Value::as_f64) + != Some(3.0) + { + return Err("unsupported specification/behavioral registry".to_string()); + } + let mut count = 0; + for profile in registry + .get("profiles") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + if profile.get("id").and_then(Value::as_str) != Some(name) { + continue; + } + count += 1; + if profile.get("version").and_then(Value::as_f64) != Some(version as f64) + || profile.get("model").and_then(Value::as_str) + != Some(format!("formal/dialcache-{name}-conformance.qnt").as_str()) + || profile.get("smoke").and_then(Value::as_str) + != Some(format!("formal/{name}-smoke.itf.json").as_str()) + { + return Err(format!("unsupported {name} profile definition/version")); + } + } + if count != 1 { + return Err(format!("registry needs exactly one {name} profile")); + } + Ok(()) +} + +/// Checks the registry entry of a behavior profile at the version this port implements. +pub fn require_behavior_profile(profile: &str) -> Result<(), String> { + let version = behavior_profile_version(profile) + .ok_or_else(|| format!("unknown behavior profile {profile}"))?; + profile_registry_check(profile, version) +} diff --git a/rust/tests/formal/json.rs b/rust/tests/formal/json.rs new file mode 100644 index 00000000..80228483 --- /dev/null +++ b/rust/tests/formal/json.rs @@ -0,0 +1,223 @@ +//! Strict JSON handling for the replay harness. +//! +//! Ports the Go harness helpers `validateBehaviorJSON`, `behaviorITF`, `bequal` +//! and `behaviorKeys`. Generated histories, coordinator replies and evidence +//! files are trust boundaries: an ambiguous document (duplicate member names, +//! non-finite or unsafe integers, trailing content) is rejected instead of being +//! decoded with last-key-wins or silent rounding. + +use serde_json::{Map, Value}; +use std::collections::HashSet; + +/// Largest integer JavaScript represents exactly; ITF integers outside +/// `[-MAX_SAFE_INTEGER, MAX_SAFE_INTEGER]` are rejected. +pub const MAX_SAFE_INTEGER: i64 = 9_007_199_254_740_991; + +/// Parses `text` as JSON, rejecting duplicate object member names (compared +/// after unescaping), non-finite numbers and trailing content. +pub fn strict_parse(text: &str) -> Result { + let value: Value = + serde_json::from_str(text).map_err(|error| format!("invalid JSON: {error}"))?; + check_duplicate_keys(text.as_bytes())?; + check_finite(&value)?; + Ok(value) +} + +/// Decodes every `{"#bigint":"..."}` object into a JSON number after checking +/// it is a safe integer, and rejects NaN, infinities and unsafe plain numbers. +pub fn decode_itf(value: Value) -> Result { + match value { + Value::Object(map) => { + if let Some(raw) = map.get("#bigint") { + if map.len() != 1 { + return Err("malformed ITF integer".to_string()); + } + let parsed = raw.as_str().and_then(|text| text.parse::().ok()); + return match parsed { + Some(n) if (-MAX_SAFE_INTEGER..=MAX_SAFE_INTEGER).contains(&n) => { + Ok(Value::from(n)) + } + _ => Err("unsafe ITF integer".to_string()), + }; + } + let mut out = Map::new(); + for (key, item) in map { + out.insert(key, decode_itf(item)?); + } + Ok(Value::Object(out)) + } + Value::Array(items) => items + .into_iter() + .map(decode_itf) + .collect::, _>>() + .map(Value::Array), + Value::Number(ref number) => match number.as_f64() { + Some(n) if n.is_finite() && n.abs() <= MAX_SAFE_INTEGER as f64 => Ok(value), + _ => Err("unsafe JSON number".to_string()), + }, + other => Ok(other), + } +} + +/// Compares two JSON values, treating numbers numerically (`1 == 1.0`) and +/// everything else structurally. `null`, `false`, `0`, `""` and `[]` are all +/// distinct from each other. +pub fn json_equal(a: &Value, b: &Value) -> bool { + match (a, b) { + (Value::Null, Value::Null) => true, + (Value::Bool(x), Value::Bool(y)) => x == y, + (Value::Number(x), Value::Number(y)) => match (x.as_f64(), y.as_f64()) { + (Some(x), Some(y)) => x == y, + _ => false, + }, + (Value::String(x), Value::String(y)) => x == y, + (Value::Array(x), Value::Array(y)) => { + x.len() == y.len() && x.iter().zip(y).all(|(a, b)| json_equal(a, b)) + } + (Value::Object(x), Value::Object(y)) => { + x.len() == y.len() + && x.iter() + .all(|(key, value)| y.get(key).is_some_and(|other| json_equal(value, other))) + } + _ => false, + } +} + +/// Joins an object's member names in sorted order with commas, like the Go +/// harness's `behaviorKeys`; used for exact envelope and reply shape checks. +pub fn sorted_keys(object: &Map) -> String { + let mut keys: Vec<&str> = object.keys().map(String::as_str).collect(); + keys.sort_unstable(); + keys.join(",") +} + +/// Sorted keys of `value` when it is an object, otherwise the empty string. +pub fn keys_of(value: &Value) -> String { + value.as_object().map(sorted_keys).unwrap_or_default() +} + +fn check_finite(value: &Value) -> Result<(), String> { + match value { + Value::Number(number) => match number.as_f64() { + Some(n) if n.is_finite() => Ok(()), + _ => Err("non-finite JSON number".to_string()), + }, + Value::Array(items) => items.iter().try_for_each(check_finite), + Value::Object(map) => map.values().try_for_each(check_finite), + _ => Ok(()), + } +} + +/// Second walk over syntactically valid JSON that checks duplicate object member +/// names without allocating every scalar. Escaped names are decoded before +/// comparison so `"a"` and its `\u0061` escape spell the same member name. +fn check_duplicate_keys(raw: &[u8]) -> Result<(), String> { + let mut walker = Walker { raw, at: 0 }; + walker.walk() +} + +struct Walker<'a> { + raw: &'a [u8], + at: usize, +} + +impl Walker<'_> { + fn peek(&self) -> Result { + self.raw + .get(self.at) + .copied() + .ok_or_else(|| "truncated JSON".to_string()) + } + + fn space(&mut self) { + while self.at < self.raw.len() && matches!(self.raw[self.at], b' ' | b'\n' | b'\r' | b'\t') + { + self.at += 1; + } + } + + fn quoted(&mut self) -> Result { + let start = self.at; + self.at += 1; + let mut escaped = false; + loop { + match self.peek()? { + b'"' => break, + b'\\' => { + escaped = true; + self.at += 2; + } + _ => self.at += 1, + } + } + self.at += 1; + let slice = &self.raw[start..self.at]; + if !escaped { + return std::str::from_utf8(&slice[1..slice.len() - 1]) + .map(str::to_owned) + .map_err(|error| error.to_string()); + } + let text = std::str::from_utf8(slice).map_err(|error| error.to_string())?; + serde_json::from_str::(text).map_err(|error| error.to_string()) + } + + fn walk(&mut self) -> Result<(), String> { + self.space(); + match self.peek()? { + b'{' => { + self.at += 1; + self.space(); + if self.peek()? == b'}' { + self.at += 1; + return Ok(()); + } + let mut seen: HashSet = HashSet::new(); + loop { + self.space(); + let key = self.quoted()?; + if !seen.insert(key.clone()) { + return Err(format!("duplicate JSON key {key:?}")); + } + self.space(); + self.at += 1; // ':' + self.walk()?; + self.space(); + if self.peek()? == b'}' { + self.at += 1; + return Ok(()); + } + self.at += 1; // ',' + } + } + b'[' => { + self.at += 1; + self.space(); + if self.peek()? == b']' { + self.at += 1; + return Ok(()); + } + loop { + self.walk()?; + self.space(); + if self.peek()? == b']' { + self.at += 1; + return Ok(()); + } + self.at += 1; // ',' + } + } + b'"' => self.quoted().map(|_| ()), + _ => { + while self.at < self.raw.len() + && !matches!( + self.raw[self.at], + b',' | b'}' | b']' | b' ' | b'\n' | b'\r' | b'\t' + ) + { + self.at += 1; + } + Ok(()) + } + } + } +} diff --git a/rust/tests/formal/mod.rs b/rust/tests/formal/mod.rs new file mode 100644 index 00000000..5f08ebf3 --- /dev/null +++ b/rust/tests/formal/mod.rs @@ -0,0 +1,16 @@ +//! Language-neutral replay infrastructure shared by the Rust conformance +//! binaries: strict JSON, the protocol schema interpreter, the coordinator +//! transport, corpus/inventory discovery, the JSONL report and the witness +//! evidence check. Cache drivers plug into [`transport::Coordinator::execute`]. +//! +//! Each `tests/*.rs` binary includes this directory as a module and uses a +//! different subset of it, so unused-item lints are silenced here rather than +//! per binary. +#![allow(dead_code)] + +pub mod inventory; +pub mod json; +pub mod report; +pub mod schema; +pub mod transport; +pub mod witness; diff --git a/rust/tests/formal/report.rs b/rust/tests/formal/report.rs new file mode 100644 index 00000000..8da4bf7f --- /dev/null +++ b/rust/tests/formal/report.rs @@ -0,0 +1,168 @@ +//! JSONL assertion report of one Rust conformance run. +//! +//! The Node completion adapter reads this file to credit each inventory id, so +//! every record is one JSON object per line: a `start` header, one `case` per +//! required id, and a `finish` footer with the totals. The path comes from +//! `DIALCACHE_RUST_REPORT`; without it the writer is a no-op that still counts. + +use serde_json::{json, Map, Value}; +use std::fs::File; +use std::io::{BufWriter, Write}; +use std::path::Path; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// Current wall time in epoch milliseconds. +pub fn now_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|elapsed| elapsed.as_millis() as i64) + .unwrap_or(0) +} + +/// Totals of one run. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Summary { + /// Cases reported. + pub cases: u64, + /// Cases that failed. + pub failed: u64, +} + +impl Summary { + /// `passed` when nothing failed, otherwise `failed`. + pub fn status(&self) -> &'static str { + if self.failed == 0 { + "passed" + } else { + "failed" + } + } +} + +/// JSONL report writer. +pub struct Report { + output: Option>, + summary: Summary, + failures: Vec<(String, String)>, +} + +impl std::fmt::Debug for Report { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Report") + .field("enabled", &self.output.is_some()) + .field("summary", &self.summary) + .finish() + } +} + +impl Report { + /// Creates the report at `DIALCACHE_RUST_REPORT`, or a counting no-op when unset. + pub fn from_env() -> Result { + let path = std::env::var("DIALCACHE_RUST_REPORT") + .ok() + .filter(|value| !value.is_empty()); + Report::create(path.as_deref().map(Path::new)) + } + + /// Creates (truncates) the report file, or a counting no-op for `None`. + pub fn create(path: Option<&Path>) -> Result { + let output = match path { + Some(path) => Some(BufWriter::new( + File::create(path).map_err(|error| format!("{}: {error}", path.display()))?, + )), + None => None, + }; + Ok(Report { + output, + summary: Summary::default(), + failures: Vec::new(), + }) + } + + /// Whether records are written anywhere. + pub fn enabled(&self) -> bool { + self.output.is_some() + } + + fn write(&mut self, record: Value) -> Result<(), String> { + if let Some(output) = self.output.as_mut() { + serde_json::to_writer(&mut *output, &record).map_err(|error| error.to_string())?; + output.write_all(b"\n").map_err(|error| error.to_string())?; + } + Ok(()) + } + + /// Writes the `start` header. + pub fn start(&mut self) -> Result<(), String> { + self.write(json!({"schemaVersion": 1, "kind": "start", "implementation": "rust", "startedAt": now_ms()})) + } + + /// Records one case with its outcome and timestamps. + pub fn case( + &mut self, + id: &str, + result: &Result<(), String>, + started_ms: i64, + finished_ms: i64, + ) -> Result<(), String> { + self.summary.cases += 1; + let mut record = Map::new(); + record.insert("kind".to_string(), Value::from("case")); + record.insert("id".to_string(), Value::from(id)); + record.insert( + "status".to_string(), + Value::from(if result.is_ok() { "passed" } else { "failed" }), + ); + record.insert("startedAt".to_string(), Value::from(started_ms)); + record.insert("finishedAt".to_string(), Value::from(finished_ms)); + if let Err(message) = result { + self.summary.failed += 1; + self.failures.push((id.to_string(), message.clone())); + record.insert("message".to_string(), Value::from(message.as_str())); + } + self.write(Value::Object(record)) + } + + /// Runs `body` as one timed case and records it. + pub fn run_case( + &mut self, + id: &str, + body: impl FnOnce() -> Result<(), String>, + ) -> Result<(), String> { + let started = now_ms(); + let result = body(); + let finished = now_ms(); + self.case(id, &result, started, finished)?; + result + } + + /// Totals so far. + pub fn summary(&self) -> Summary { + self.summary + } + + /// Writes the `finish` footer, flushes, and prints a human summary. + pub fn finish(&mut self) -> Result { + let summary = self.summary; + self.write(json!({ + "kind": "finish", + "status": summary.status(), + "finishedAt": now_ms(), + "cases": summary.cases, + "failed": summary.failed, + }))?; + if let Some(output) = self.output.as_mut() { + output.flush().map_err(|error| error.to_string())?; + } + for (id, message) in &self.failures { + println!("FAILED {id}: {message}"); + } + println!( + "rust conformance: {} cases, {} failed, status {}", + summary.cases, + summary.failed, + summary.status() + ); + Ok(summary) + } +} diff --git a/rust/tests/formal/schema.rs b/rust/tests/formal/schema.rs new file mode 100644 index 00000000..edd44e1c --- /dev/null +++ b/rust/tests/formal/schema.rs @@ -0,0 +1,380 @@ +//! Local interpreter for `formal/replay/protocol.schema.json`. +//! +//! Ports `readReplaySchema`, `validateReplaySchema`, `matchesReplaySchema`, +//! `replaySchemaType` and `replayObservationError` from the Go harness. Each +//! port interprets the schema mechanics; the shared schema owns the command and +//! observation field lists, so no second copy of them lives here. + +use super::json::{json_equal, keys_of, strict_parse, MAX_SAFE_INTEGER}; +use serde_json::{Map, Value}; +use std::path::PathBuf; + +/// Repository-relative location of the shared protocol schema. +pub const SCHEMA_PATH: &str = "formal/replay/protocol.schema.json"; + +/// Observation definitions a prepare result may name. Every observation the +/// driver reports is validated against the named definition before it is sent. +pub const OBSERVATION_DEFINITIONS: [&str; 3] = [ + "behaviorObservation", + "coreObservation", + "localClockObservation", +]; + +/// Schema keywords the interpreter understands; any other keyword is rejected at +/// load so an unknown constraint cannot be silently ignored. +const KEYWORDS: [&str; 20] = [ + "$schema", + "$id", + "$defs", + "$ref", + "title", + "description", + "oneOf", + "anyOf", + "const", + "enum", + "type", + "properties", + "required", + "additionalProperties", + "items", + "minItems", + "minimum", + "maximum", + "minLength", + "pattern", +]; + +/// The loaded protocol schema. +#[derive(Debug, Clone)] +pub struct Schema { + root: Map, +} + +/// Absolute path of the shared schema, resolved from this crate's manifest. +pub fn schema_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(SCHEMA_PATH) +} + +impl Schema { + /// Reads and validates `../formal/replay/protocol.schema.json`. + pub fn load() -> Result { + let path = schema_path(); + let text = std::fs::read_to_string(&path) + .map_err(|error| format!("{}: {error}", path.display()))?; + Schema::parse(&text) + } + + /// Parses schema text, rejecting unsupported keywords and patterns. + pub fn parse(text: &str) -> Result { + let root = match strict_parse(text)? { + Value::Object(root) => root, + _ => return Err("replay schema is not an object".to_string()), + }; + validate_schema_keywords(&root)?; + validate_patterns(&root)?; + if !root.get("$defs").is_some_and(Value::is_object) { + return Err("replay schema lacks $defs".to_string()); + } + Ok(Schema { root }) + } + + /// The `$defs` table every `$ref` resolves against. + pub fn defs(&self) -> &Map { + self.root + .get("$defs") + .and_then(Value::as_object) + .expect("validated at load") + } + + /// Looks up one `$defs` definition. + pub fn definition(&self, name: &str) -> Option<&Map> { + self.defs().get(name).and_then(Value::as_object) + } + + /// Whether `value` satisfies the named `$defs` definition. + pub fn matches_definition(&self, value: &Value, name: &str) -> bool { + self.definition(name) + .is_some_and(|rule| matches(value, rule, self.defs())) + } + + /// Validates one driver observation against the definition `prepare` named. + pub fn observation_error(&self, observed: &Value, definition: &str) -> Result<(), String> { + observation_error(observed, definition, self.defs()) + } + + /// Whether every input is a well-formed `$defs/command`. + pub fn commands_valid(&self, inputs: &[Value]) -> bool { + commands_valid(inputs, self.defs()) + } +} + +/// Rejects any keyword outside the supported list, recursively through +/// `$defs`, `properties`, `oneOf`, `anyOf`, `items` and `additionalProperties`. +pub fn validate_schema_keywords(rule: &Map) -> Result<(), String> { + for key in rule.keys() { + if !KEYWORDS.contains(&key.as_str()) { + return Err(format!("unsupported replay schema keyword: {key}")); + } + } + for child in children(rule) { + validate_schema_keywords(child)?; + } + Ok(()) +} + +fn validate_patterns(rule: &Map) -> Result<(), String> { + if let Some(pattern) = rule.get("pattern") { + match pattern.as_str() { + Some(text) if pattern_supported(text) => {} + _ => return Err(format!("unsupported replay schema pattern: {pattern}")), + } + } + for child in children(rule) { + validate_patterns(child)?; + } + Ok(()) +} + +fn children(rule: &Map) -> Vec<&Map> { + let mut out = Vec::new(); + for key in ["$defs", "properties"] { + if let Some(table) = rule.get(key).and_then(Value::as_object) { + out.extend(table.values().filter_map(Value::as_object)); + } + } + for key in ["oneOf", "anyOf"] { + if let Some(options) = rule.get(key).and_then(Value::as_array) { + out.extend(options.iter().filter_map(Value::as_object)); + } + } + for key in ["items", "additionalProperties"] { + if let Some(child) = rule.get(key).and_then(Value::as_object) { + out.push(child); + } + } + out +} + +/// Whether `value` satisfies `rule`, resolving `#/$defs/` references in `defs`. +pub fn matches(value: &Value, rule: &Map, defs: &Map) -> bool { + if let Some(reference) = rule.get("$ref").and_then(Value::as_str) { + let name = reference.strip_prefix("#/$defs/").unwrap_or(reference); + return match defs.get(name) { + Some(Value::Object(target)) => matches(value, target, defs), + Some(_) => matches(value, &Map::new(), defs), + None => false, + }; + } + if let Some(options) = rule.get("oneOf").and_then(Value::as_array) { + let count = options + .iter() + .filter(|option| matches(value, as_rule(option), defs)) + .count(); + if count != 1 { + return false; + } + } + if let Some(options) = rule.get("anyOf").and_then(Value::as_array) { + if !options + .iter() + .any(|option| matches(value, as_rule(option), defs)) + { + return false; + } + } + if let Some(constant) = rule.get("const") { + if !json_equal(value, constant) { + return false; + } + } + if let Some(options) = rule.get("enum").and_then(Value::as_array) { + if !options.iter().any(|option| json_equal(value, option)) { + return false; + } + } + if let Some(expected) = rule.get("type") { + let matched = match expected { + Value::String(name) => schema_type(value, name), + Value::Array(names) => names + .iter() + .any(|name| schema_type(value, name.as_str().unwrap_or(""))), + _ => false, + }; + if !matched { + return false; + } + } + match value { + Value::Number(number) => { + let n = number.as_f64().unwrap_or(f64::NAN); + if let Some(minimum) = rule.get("minimum").and_then(Value::as_f64) { + if n < minimum { + return false; + } + } + if let Some(maximum) = rule.get("maximum").and_then(Value::as_f64) { + if n > maximum { + return false; + } + } + } + Value::String(text) => { + if let Some(minimum) = rule.get("minLength").and_then(Value::as_f64) { + if (text.chars().count() as f64) < minimum.trunc() { + return false; + } + } + if let Some(pattern) = rule.get("pattern") { + if !pattern + .as_str() + .is_some_and(|pattern| pattern_matches(pattern, text)) + { + return false; + } + } + } + Value::Array(items) => { + if let Some(minimum) = rule.get("minItems").and_then(Value::as_f64) { + if (items.len() as f64) < minimum.trunc() { + return false; + } + } + if let Some(schema) = rule.get("items").and_then(Value::as_object) { + if !items.iter().all(|item| matches(item, schema, defs)) { + return false; + } + } + } + Value::Object(members) => { + if let Some(required) = rule.get("required").and_then(Value::as_array) { + if !required + .iter() + .all(|key| members.contains_key(key.as_str().unwrap_or(""))) + { + return false; + } + } + let empty = Map::new(); + let properties = rule + .get("properties") + .and_then(Value::as_object) + .unwrap_or(&empty); + let additional = rule.get("additionalProperties"); + for (key, item) in members { + if let Some(property) = properties.get(key) { + if !matches(item, as_rule(property), defs) { + return false; + } + } else if additional == Some(&Value::Bool(false)) { + return false; + } else if let Some(schema) = additional.and_then(Value::as_object) { + if !matches(item, schema, defs) { + return false; + } + } + } + } + _ => {} + } + true +} + +fn as_rule(value: &Value) -> &Map { + static EMPTY: std::sync::OnceLock> = std::sync::OnceLock::new(); + value + .as_object() + .unwrap_or_else(|| EMPTY.get_or_init(Map::new)) +} + +/// JSON Schema primitive type check; `integer` means a finite integral number. +pub fn schema_type(value: &Value, expected: &str) -> bool { + match expected { + "null" => value.is_null(), + "object" => value.is_object(), + "array" => value.is_array(), + "string" => value.is_string(), + "boolean" => value.is_boolean(), + "integer" | "number" => match value.as_f64() { + Some(n) => n.is_finite() && (expected == "number" || n.trunc() == n), + None => false, + }, + _ => false, + } +} + +/// Validates one driver observation against the `$defs` definition the prepare +/// result named. The diagnostic names the record's keys, never its values, and +/// carries no `expected:`/`actual:` markers: a shape defect is infrastructure +/// evidence, not a mutation detection. +pub fn observation_error( + observed: &Value, + definition: &str, + defs: &Map, +) -> Result<(), String> { + let target = match defs.get(definition) { + Some(Value::Object(target)) if OBSERVATION_DEFINITIONS.contains(&definition) => target, + _ => { + return Err(format!( + "unknown replay observation definition {definition:?}" + )) + } + }; + if !matches(observed, target, defs) { + return Err(format!( + "driver produced a malformed {definition} observation: keys [{}]", + keys_of(observed) + )); + } + Ok(()) +} + +/// Whether every input is a well-formed `$defs/command`. +pub fn commands_valid(inputs: &[Value], defs: &Map) -> bool { + match defs.get("command") { + Some(Value::Object(command)) => inputs.iter().all(|input| matches(input, command, defs)), + _ => false, + } +} + +/// Whether `value` is a whole number in `[minimum, MAX_SAFE_INTEGER]`, the +/// check the transport applies to step counts and indices. +pub fn safe_index(value: Option<&Value>, minimum: i64) -> bool { + match value.and_then(Value::as_f64) { + Some(n) => n >= minimum as f64 && n <= MAX_SAFE_INTEGER as f64 && n.trunc() == n, + None => false, + } +} + +// The schema uses exactly two regular expressions. They are matched by hand so +// the harness needs no regex dependency; `validate_patterns` fails loading if +// the schema ever gains another one. +const POSITIVE_INTEGER: &str = "^[1-9][0-9]*$"; +const CALL_ERROR: &str = "^(source|timeout):(0|[1-9][0-9]*)$|^unexpected:"; + +fn pattern_supported(pattern: &str) -> bool { + pattern == POSITIVE_INTEGER || pattern == CALL_ERROR +} + +fn pattern_matches(pattern: &str, text: &str) -> bool { + match pattern { + POSITIVE_INTEGER => positive_integer(text), + CALL_ERROR => { + if text.starts_with("unexpected:") { + return true; + } + ["source:", "timeout:"] + .iter() + .filter_map(|prefix| text.strip_prefix(prefix)) + .any(|rest| rest == "0" || positive_integer(rest)) + } + _ => false, + } +} + +fn positive_integer(text: &str) -> bool { + let mut chars = text.chars(); + matches!(chars.next(), Some('1'..='9')) && chars.all(|c| c.is_ascii_digit()) +} diff --git a/rust/tests/formal/transport.rs b/rust/tests/formal/transport.rs new file mode 100644 index 00000000..d92c74ee --- /dev/null +++ b/rust/tests/formal/transport.rs @@ -0,0 +1,490 @@ +//! JSON-lines client for the shared Node replay coordinator. +//! +//! Ports `replayCoordinator` from `go/replay_coordinator_test.go`: one request +//! per line on the child's stdin, one reply per line on its stdout, a strictly +//! increasing request id, exact envelope checks and a real-time watchdog that +//! kills a child which leaves a request pending longer than the timeout. +//! +//! The coordinator is `node /formal/replay/coordinator.mjs`, started with +//! the `node` found on `PATH`. **It must be Node 24** (the shared replay code +//! relies on it); when running these tests on a machine whose default `node` +//! is another major version, prepend the Node 24 `bin` directory to `PATH` +//! (for example `/opt/homebrew/opt/node@24/bin`) before invoking `cargo test`. +//! Node is test tooling only; the cache library itself has no Node dependency. +//! +//! Everything here is synchronous (`std::process`, `std::io`, threads). The +//! watchdog observes real process time only; replay IO never consumes a driver +//! deadline or releases a cache gate. + +use super::json::{sorted_keys, strict_parse}; +use super::schema::{safe_index, Schema, OBSERVATION_DEFINITIONS}; +use serde_json::{Map, Value}; +use std::io::{BufRead, BufReader, Read, Write}; +use std::path::{Path, PathBuf}; +use std::process::{Child, ChildStdin, ChildStdout, Command, Stdio}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; + +/// The settlement contract every observe request declares. +pub const SETTLEMENT: &str = "causally-ready-v1"; + +/// Largest reply frame the transport buffers. +pub const FRAME_LIMIT: usize = 64 * 1024 * 1024; + +/// Default real-time bound on one coordinator round trip. +pub const DEFAULT_TIMEOUT: Duration = Duration::from_secs(30); + +/// Repository-relative location of the shared coordinator program. +pub const COORDINATOR_PATH: &str = "formal/replay/coordinator.mjs"; + +/// Absolute path of `formal/replay/coordinator.mjs`, resolved from this crate's manifest. +pub fn coordinator_program() -> Result { + let relative = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join(COORDINATOR_PATH); + std::path::absolute(&relative).map_err(|error| format!("{}: {error}", relative.display())) +} + +/// A validated `prepare` result: the coordinator's session for one history. +#[derive(Debug, Clone, PartialEq)] +pub struct Prepared { + /// Session identifier for the following `observe`/`discard` requests. + pub session: String, + /// Number of steps, including the initial observation at index 0. + pub steps: i64, + /// `$defs` definition every observation of this session must satisfy. + pub observation: String, + /// Initialization fixture for the native driver (an object). + pub fixture: Value, + /// Commands to apply before the first observation. + pub setup: Vec, + /// Action name of every step; the first is always `init`. + pub actions: Vec, +} + +/// One running coordinator process and its JSON-lines connection. +pub struct Coordinator { + schema: Schema, + child: Arc>, + input: Option, + output: BufReader, + stderr: Option>>, + watchdog: Option>, + pending: Arc>>, + stopped: Arc, + sequence: i64, + expect_process_failure: bool, + finished: bool, +} + +impl std::fmt::Debug for Coordinator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Coordinator") + .field("sequence", &self.sequence) + .field("finished", &self.finished) + .finish_non_exhaustive() + } +} + +impl Coordinator { + /// Starts the shared coordinator (`node formal/replay/coordinator.mjs`) + /// with the default 30 s request timeout. + pub fn spawn() -> Result { + let mut command = Command::new("node"); + command.arg(coordinator_program()?); + Coordinator::start(command, DEFAULT_TIMEOUT, false) + } + + /// Starts an arbitrary child as the coordinator. Test controls substitute + /// a broken transport this way without replacing any cache implementation; + /// `expect_process_failure` silences the exit-status check in [`finish`]. + /// + /// [`finish`]: Coordinator::finish + pub fn start( + mut command: Command, + timeout: Duration, + expect_process_failure: bool, + ) -> Result { + let schema = Schema::load()?; + command + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let mut child = command + .spawn() + .map_err(|error| format!("cannot start replay coordinator: {error}"))?; + let input = child.stdin.take().ok_or("coordinator stdin unavailable")?; + let output = child + .stdout + .take() + .ok_or("coordinator stdout unavailable")?; + let mut errors = child + .stderr + .take() + .ok_or("coordinator stderr unavailable")?; + let stderr = std::thread::spawn(move || { + let mut captured = Vec::new(); + let _ = errors.read_to_end(&mut captured); + captured + }); + let child = Arc::new(Mutex::new(child)); + let pending: Arc>> = Arc::new(Mutex::new(None)); + let stopped = Arc::new(AtomicBool::new(false)); + let watchdog = { + let (child, pending, stopped) = ( + Arc::clone(&child), + Arc::clone(&pending), + Arc::clone(&stopped), + ); + std::thread::spawn(move || { + while !stopped.load(Ordering::SeqCst) { + std::thread::sleep(Duration::from_millis(10)); + let expired = pending + .lock() + .map(|since| since.is_some_and(|since| since.elapsed() > timeout)) + .unwrap_or(false); + if expired { + if let Ok(mut child) = child.lock() { + let _ = child.kill(); + } + return; + } + } + }) + }; + Ok(Coordinator { + schema, + child, + input: Some(input), + output: BufReader::new(output), + stderr: Some(stderr), + watchdog: Some(watchdog), + pending, + stopped, + sequence: 0, + expect_process_failure, + finished: false, + }) + } + + /// The protocol schema this transport validates against. + pub fn schema(&self) -> &Schema { + &self.schema + } + + fn arm(&self, since: Option) { + if let Ok(mut pending) = self.pending.lock() { + *pending = since; + } + } + + /// Sends one request, adding `version: 1` and the next id, and returns the + /// validated `result` object of a successful reply. + pub fn call(&mut self, mut request: Map) -> Result { + self.sequence += 1; + request.insert("version".to_string(), Value::from(1)); + request.insert("id".to_string(), Value::from(self.sequence)); + let mut raw = + serde_json::to_vec(&Value::Object(request)).map_err(|error| error.to_string())?; + raw.push(b'\n'); + self.arm(Some(Instant::now())); + let exchanged = self.exchange(&raw); + self.arm(None); + let line = exchanged?; + let text = std::str::from_utf8(&line) + .map_err(|_| "coordinator response is not UTF-8".to_string())?; + let envelope = match strict_parse(text)? { + Value::Object(envelope) => envelope, + _ => return Err("malformed coordinator envelope".to_string()), + }; + let ok = match envelope.get("ok") { + Some(Value::Bool(ok)) => *ok, + Some(_) => return Err("malformed coordinator envelope".to_string()), + None => false, + }; + let keys = sorted_keys(&envelope); + if (ok && keys != "id,ok,result,version") || (!ok && keys != "error,id,ok,version") { + return Err("malformed coordinator envelope".to_string()); + } + if !safe_index(envelope.get("version"), 1) || !safe_index(envelope.get("id"), 1) { + return Err("malformed coordinator envelope".to_string()); + } + let version = envelope + .get("version") + .and_then(Value::as_f64) + .unwrap_or(0.0); + let id = envelope.get("id").and_then(Value::as_f64).unwrap_or(0.0); + if version != 1.0 || id != self.sequence as f64 { + return Err("unknown or out-of-sequence coordinator response".to_string()); + } + if !ok { + return match envelope.get("error") { + Some(Value::String(message)) if !message.is_empty() => { + Err(format!("shared replay: {message}")) + } + _ => Err("malformed coordinator failure".to_string()), + }; + } + match envelope.get("result") { + Some(Value::Object(_)) => Ok(envelope.get("result").cloned().unwrap_or(Value::Null)), + _ => Err("malformed coordinator success".to_string()), + } + } + + fn exchange(&mut self, raw: &[u8]) -> Result, String> { + let input = self + .input + .as_mut() + .ok_or("coordinator input already closed")?; + input + .write_all(raw) + .and_then(|()| input.flush()) + .map_err(|error| format!("coordinator write failed: {error}"))?; + read_frame(&mut self.output, FRAME_LIMIT).map_err(|error| { + format!("coordinator unavailable or exceeded real-time request limit: {error}") + }) + } + + /// Prepares one history and validates the coordinator's session description. + pub fn prepare( + &mut self, + profile: &str, + path: &Path, + raw: Option<&str>, + ) -> Result { + let absolute = + std::path::absolute(path).map_err(|error| format!("{}: {error}", path.display()))?; + let mut request = Map::new(); + request.insert("op".to_string(), Value::from("prepare")); + request.insert("profile".to_string(), Value::from(profile)); + request.insert( + "path".to_string(), + Value::from(absolute.to_string_lossy().into_owned()), + ); + if let Some(raw) = raw { + request.insert("raw".to_string(), Value::from(raw)); + } + let result = self.call(request)?; + let result = result.as_object().ok_or("malformed replay preparation")?; + let session = result.get("session").and_then(Value::as_str).unwrap_or(""); + if sorted_keys(result) != "actions,fixture,observation,session,settlement,setup,steps" + || result.get("settlement").and_then(Value::as_str) != Some(SETTLEMENT) + || session.is_empty() + || !safe_index(result.get("steps"), 2) + { + return Err("malformed replay preparation".to_string()); + } + let definition = result + .get("observation") + .and_then(Value::as_str) + .unwrap_or(""); + if !OBSERVATION_DEFINITIONS.contains(&definition) + || self.schema.definition(definition).is_none() + { + return Err(format!( + "malformed replay observation definition {definition:?}" + )); + } + let fixture = match result.get("fixture") { + Some(fixture @ Value::Object(_)) => fixture.clone(), + _ => return Err("malformed replay fixture".to_string()), + }; + let setup = match result.get("setup") { + Some(Value::Array(setup)) if self.schema.commands_valid(setup) => setup.clone(), + _ => return Err("malformed replay setup".to_string()), + }; + let steps = result.get("steps").and_then(Value::as_f64).unwrap_or(0.0) as i64; + let actions = match result.get("actions") { + Some(Value::Array(actions)) if actions.len() as i64 == steps => actions, + _ => return Err("malformed replay actions".to_string()), + }; + let mut names = Vec::with_capacity(actions.len()); + for (index, action) in actions.iter().enumerate() { + let name = action.as_str().unwrap_or(""); + if name.is_empty() || (index == 0) != (name == "init") { + return Err("malformed replay action".to_string()); + } + names.push(name.to_string()); + } + Ok(Prepared { + session: session.to_string(), + steps, + observation: definition.to_string(), + fixture, + setup, + actions: names, + }) + } + + /// Replays one prepared history through a driver: applies setup, then for + /// every step runs the monitors, validates the observation locally, sends + /// it and applies the returned commands until the coordinator completes the + /// session. Any early exit discards the coordinator session. + pub fn execute( + &mut self, + prepared: &Prepared, + apply: &mut dyn FnMut(&Value) -> Result<(), String>, + observation: &mut dyn FnMut() -> Value, + wall_ms: &mut dyn FnMut() -> i64, + monitors: &mut [&mut dyn FnMut() -> Result<(), String>], + ) -> Result<(), String> { + let outcome = self.run(prepared, apply, observation, wall_ms, monitors); + if outcome.is_err() { + let mut request = Map::new(); + request.insert("op".to_string(), Value::from("discard")); + request.insert( + "session".to_string(), + Value::from(prepared.session.as_str()), + ); + let _ = self.call(request); + } + outcome + } + + fn run( + &mut self, + prepared: &Prepared, + apply: &mut dyn FnMut(&Value) -> Result<(), String>, + observation: &mut dyn FnMut() -> Value, + wall_ms: &mut dyn FnMut() -> i64, + monitors: &mut [&mut dyn FnMut() -> Result<(), String>], + ) -> Result<(), String> { + for input in &prepared.setup { + apply(input)?; + } + for index in 0..prepared.steps { + for monitor in monitors.iter_mut() { + monitor()?; + } + // A malformed record is a driver defect. Attribute it here, before + // the coordinator sees it, so no round trip or session state is + // spent on it. + let observed = observation(); + self.schema + .observation_error(&observed, &prepared.observation)?; + let mut environment = Map::new(); + environment.insert("wallMs".to_string(), Value::from(wall_ms())); + let mut request = Map::new(); + request.insert("op".to_string(), Value::from("observe")); + request.insert( + "session".to_string(), + Value::from(prepared.session.as_str()), + ); + request.insert("index".to_string(), Value::from(index)); + request.insert("settlement".to_string(), Value::from(SETTLEMENT)); + request.insert("observed".to_string(), observed); + request.insert("environment".to_string(), Value::Object(environment)); + let result = self.call(request)?; + let result = result.as_object().ok_or("malformed next replay command")?; + let next = index + 1; + if result.get("complete") == Some(&Value::Bool(true)) { + let steps = result.get("steps"); + if sorted_keys(result) != "complete,steps" + || next != prepared.steps + || !safe_index(steps, 2) + || steps.and_then(Value::as_f64) != Some(next as f64) + { + return Err("premature or malformed replay completion".to_string()); + } + return Ok(()); + } + let inputs = result.get("inputs").and_then(Value::as_array); + if sorted_keys(result) != "complete,index,inputs" + || result.get("complete") != Some(&Value::Bool(false)) + || !safe_index(result.get("index"), 1) + || result.get("index").and_then(Value::as_f64) != Some(next as f64) + || !inputs + .is_some_and(|inputs| !inputs.is_empty() && self.schema.commands_valid(inputs)) + { + return Err("malformed next replay command".to_string()); + } + for input in inputs.unwrap_or(&Vec::new()) { + apply(input)?; + } + } + Err("replay ended without completion".to_string()) + } + + /// Closes the child's stdin, waits for it to exit and reports a failed exit + /// (with its captured stderr) unless the process was expected to fail. The + /// watchdog stays armed while closing so a child that ignores EOF cannot + /// hang the suite. + pub fn finish(mut self) -> Result<(), String> { + self.close() + } + + fn close(&mut self) -> Result<(), String> { + if self.finished { + return Ok(()); + } + self.finished = true; + self.arm(Some(Instant::now())); + drop(self.input.take()); + // Poll instead of blocking in wait() so the watchdog can take the child + // handle and kill a process that ignores EOF. + let status = loop { + let polled = match self.child.lock() { + Ok(mut child) => child.try_wait().map_err(|error| error.to_string()), + Err(_) => Err("coordinator handle poisoned".to_string()), + }; + match polled { + Ok(Some(status)) => break Ok(status), + Ok(None) => std::thread::sleep(Duration::from_millis(5)), + Err(error) => break Err(error), + } + }; + self.stopped.store(true, Ordering::SeqCst); + self.arm(None); + if let Some(watchdog) = self.watchdog.take() { + let _ = watchdog.join(); + } + let stderr = self + .stderr + .take() + .and_then(|thread| thread.join().ok()) + .unwrap_or_default(); + let status = status?; + if !status.success() && !self.expect_process_failure { + return Err(format!( + "shared replay process failed: {status}\n{}", + String::from_utf8_lossy(&stderr) + )); + } + Ok(()) + } +} + +impl Drop for Coordinator { + fn drop(&mut self) { + let _ = self.close(); + } +} + +/// Reads one newline-terminated frame, failing with `oversized coordinator +/// response` before buffering more than `limit` bytes. The reader's own buffer +/// bounds each fragment, so an unbounded line never accumulates in memory. +pub fn read_frame(reader: &mut R, limit: usize) -> Result, String> { + let mut line = Vec::new(); + loop { + let (fragment, done) = { + let available = reader.fill_buf().map_err(|error| error.to_string())?; + if available.is_empty() { + return Err("coordinator closed its output before completing a frame".to_string()); + } + match available.iter().position(|byte| *byte == b'\n') { + Some(at) => (available[..=at].to_vec(), true), + None => (available.to_vec(), false), + } + }; + if line.len() + fragment.len() > limit { + return Err("oversized coordinator response".to_string()); + } + reader.consume(fragment.len()); + line.extend_from_slice(&fragment); + if done { + return Ok(line); + } + } +} diff --git a/rust/tests/formal/witness.rs b/rust/tests/formal/witness.rs new file mode 100644 index 00000000..71d24258 --- /dev/null +++ b/rust/tests/formal/witness.rs @@ -0,0 +1,386 @@ +//! Witness-evidence check. +//! +//! Ports `checkWitnessEvidenceAt`, `witnessTraceKind` and `sharedReplaySources` +//! from `go/witness_evidence_test.go`. Required reachability witnesses have one +//! evaluator shared by the language drivers (`node formal/witnesses.mjs +//! evaluate`); reusing its result requires the exact corpus and definition +//! hashes recorded in `/.json` to match this checkout byte +//! for byte. It does not replace any native execution or observation assertion. + +use super::inventory::trace_kind; +use super::json::strict_parse; +use serde::{Deserialize, Deserializer}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +/// Hex SHA-256 of a file's bytes. +pub fn file_sha256(path: &Path) -> Result { + let raw = std::fs::read(path).map_err(|error| format!("{}: {error}", path.display()))?; + Ok(sha256_hex(&raw)) +} + +/// Hex SHA-256 of `data`. +pub fn sha256_hex(data: &[u8]) -> String { + sha256::digest(data) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn null_default<'de, D: Deserializer<'de>, T: Default + Deserialize<'de>>( + deserializer: D, +) -> Result { + Option::::deserialize(deserializer).map(Option::unwrap_or_default) +} + +/// `{ path?, name?, sha256 }` fingerprint of one input or history. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, serde::Serialize)] +#[serde(default)] +pub struct Digest { + /// Repository-relative definition path (inputs). + pub path: String, + /// History file name (corpus). + pub name: String, + /// Hex SHA-256. + pub sha256: String, +} + +/// One history that earned a label. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, serde::Serialize)] +#[serde(default)] +pub struct Trace { + /// History file name. + pub name: String, + /// `sampled` or `regression`. + pub kind: String, + /// Step indices at which the classifier credited the label. + #[serde(deserialize_with = "null_default")] + pub checkpoints: Vec, +} + +/// Per-label provenance (schema 2). +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, serde::Serialize)] +#[serde(default)] +pub struct Label { + /// Sampled histories that earned the label. + pub sampled: i64, + /// Exported regressions that earned the label. + pub regression: i64, + /// The citing histories. + #[serde(deserialize_with = "null_default")] + pub traces: Vec, +} + +/// The evidence document written by `node formal/witnesses.mjs evaluate`. +#[derive(Debug, Clone, Default, PartialEq, Eq, Deserialize, serde::Serialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Evidence { + pub schema_version: i64, + pub profile: String, + pub traces: i64, + #[serde(deserialize_with = "null_default")] + pub required: Vec, + #[serde(deserialize_with = "null_default")] + pub seen: Vec, + /// `None` when the document lacks per-label provenance entirely. + pub labels: Option>, + #[serde(deserialize_with = "null_default")] + pub inputs: Vec, + #[serde(deserialize_with = "null_default")] + pub corpus: Vec, +} + +fn read_strict(path: &Path) -> Result { + let text = + std::fs::read_to_string(path).map_err(|error| format!("{}: {error}", path.display()))?; + strict_parse(&text).map_err(|error| format!("{}: {error}", path.display())) +} + +fn decode Deserialize<'de>>(path: &Path) -> Result { + serde_json::from_value(read_strict(path)?) + .map_err(|error| format!("{}: {error}", path.display())) +} + +/// Checks `/.json` against the registry, definitions and +/// corpus of the checkout at `root`, where `paths` are the histories this run +/// replays for the profile. +pub fn check_witness_evidence( + root: &Path, + profile: &str, + directory: &Path, + paths: &[PathBuf], +) -> Result<(), String> { + let evidence: Evidence = decode(&directory.join(format!("{profile}.json")))?; + if evidence.schema_version != 2 + || evidence.profile != profile + || evidence.traces != paths.len() as i64 + || evidence.corpus.len() != paths.len() + { + return Err(format!("unsupported/incomplete {profile} witness evidence")); + } + let registry: HashMap> = + decode(&root.join("formal/coverage-witnesses.json"))?; + let required = registry.get(profile).cloned().unwrap_or_default(); + if required.is_empty() || required != evidence.required { + return Err(format!("{profile} required witness registry differs")); + } + let mut seen = std::collections::HashSet::new(); + for name in &evidence.seen { + if !seen.insert(name.as_str()) { + return Err(format!("duplicate witness {name}")); + } + } + for name in &required { + if !seen.contains(name.as_str()) { + return Err(format!("{profile} missing witness {name}")); + } + } + // Every required label names the histories that earned it: at least one + // sampled history or exported regression, each part of the bound corpus + // with the kind its directory gives it and cited at a checkpoint, and the + // split counts must agree with the cited histories. + let labels = evidence + .labels + .as_ref() + .ok_or_else(|| format!("{profile} witness evidence lacks per-label provenance"))?; + let corpus_kinds: HashMap = paths + .iter() + .map(|path| (base_name(path), trace_kind(path))) + .collect(); + for name in &required { + let label = match labels.get(name) { + Some(label) if label.sampled + label.regression >= 1 => label, + _ => return Err(format!("{profile} witness {name} lacks provenance")), + }; + let (mut sampled, mut regression) = (0i64, 0i64); + for trace in &label.traces { + let kind = *corpus_kinds.get(&trace.name).ok_or_else(|| { + format!( + "{profile} witness {name} cites an unknown history {}", + trace.name + ) + })?; + if trace.kind != kind { + return Err(format!( + "{profile} witness {name} reports {kind} history {} as {}", + trace.name, trace.kind + )); + } + if trace.checkpoints.is_empty() { + return Err(format!( + "{profile} witness {name} cites {} without a checkpoint", + trace.name + )); + } + if kind == "sampled" { + sampled += 1; + } else { + regression += 1; + } + } + if sampled != label.sampled || regression != label.regression { + return Err(format!( + "{profile} witness {name} counts {} sampled and {} regression hits but cites {sampled} and {regression}", + label.sampled, label.regression + )); + } + } + // The evidence binds language-neutral definitions only: the registry, the + // required witnesses, the execution manifest, the profile's model and the + // observation library, then every Quint library, the shared replay closure + // (which holds the witness classifiers) and the profile's witness sources. + let mut expected: Vec = [ + "formal/profiles.json".to_string(), + "formal/coverage-witnesses.json".to_string(), + "formal/execution.json".to_string(), + format!("formal/dialcache-{profile}-conformance.qnt"), + "formal/conformance-observations.qnt".to_string(), + ] + .to_vec(); + let execution = read_strict(&root.join("formal/execution.json"))?; + let definitions = read_strict(&root.join("formal/profiles.json"))?; + let shared = shared_replay_sources(root)?; + let mut additional: Vec = strings_at(&execution, "libraries")?; + additional.extend(shared); + for definition in definitions + .get("profiles") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + { + if definition.get("id").and_then(serde_json::Value::as_str) == Some(profile) { + additional.extend(strings_at(definition, "witnessSources")?); + } + } + for path in additional { + if !expected.contains(&path) { + expected.push(path); + } + } + if evidence.inputs.len() != expected.len() { + return Err("incomplete witness definition fingerprints".to_string()); + } + for (item, path) in evidence.inputs.iter().zip(&expected) { + if &item.path != path { + return Err(format!("unexpected witness input {}", item.path)); + } + if file_sha256(&root.join(path))? != item.sha256 { + return Err(format!("stale witness definition {path}")); + } + } + let mut actual: HashMap = HashMap::new(); + for path in paths { + let name = base_name(path); + if actual.contains_key(&name) { + return Err(format!("duplicate trace name {name}")); + } + actual.insert(name, file_sha256(path)?); + } + for item in &evidence.corpus { + match actual.remove(&item.name) { + Some(hash) if hash == item.sha256 => {} + _ => return Err(format!("{profile} witness corpus differs at {}", item.name)), + } + } + if !actual.is_empty() { + return Err("unaccounted replay traces".to_string()); + } + Ok(()) +} + +fn strings_at(value: &serde_json::Value, key: &str) -> Result, String> { + match value.get(key) { + None | Some(serde_json::Value::Null) => Ok(Vec::new()), + Some(serde_json::Value::Array(items)) => items + .iter() + .map(|item| { + item.as_str() + .map(str::to_owned) + .ok_or_else(|| format!("{key} must list strings")) + }) + .collect(), + Some(_) => Err(format!("{key} must be an array")), + } +} + +fn base_name(path: &Path) -> String { + path.file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default() +} + +/// Matches `replaySources` of `formal/profiles.json` against the `.mjs`, +/// `.mts` and `.json` files below `formal/replay`, sorted, and returns them. +pub fn shared_replay_sources(root: &Path) -> Result, String> { + let registry = read_strict(&root.join("formal/profiles.json"))?; + let declared = strings_at(®istry, "replaySources")?; + let mut actual = Vec::new(); + walk(root, &root.join("formal/replay"), &mut actual)?; + actual.sort(); + if actual.is_empty() || declared != actual { + return Err("shared replay source inventory differs from formal/replay".to_string()); + } + Ok(actual) +} + +fn walk(root: &Path, directory: &Path, out: &mut Vec) -> Result<(), String> { + let entries = std::fs::read_dir(directory) + .map_err(|error| format!("{}: {error}", directory.display()))?; + for entry in entries { + let entry = entry.map_err(|error| format!("{}: {error}", directory.display()))?; + let path = entry.path(); + if path.is_dir() { + walk(root, &path, out)?; + continue; + } + if matches!( + path.extension().and_then(|extension| extension.to_str()), + Some("mjs" | "mts" | "json") + ) { + let relative = path + .strip_prefix(root) + .map_err(|error| format!("{}: {error}", path.display()))?; + out.push( + relative + .components() + .map(|part| part.as_os_str().to_string_lossy().into_owned()) + .collect::>() + .join("/"), + ); + } + } + Ok(()) +} + +/// Self-contained SHA-256 (FIPS 180-4), kept private so the harness needs no +/// extra dependency for file fingerprints. +mod sha256 { + const K: [u32; 64] = [ + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, + 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, + 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, + 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, + 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, + 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, + 0xc67178f2, + ]; + + pub fn digest(data: &[u8]) -> [u8; 32] { + let mut state: [u32; 8] = [ + 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, + 0x5be0cd19, + ]; + let mut message = data.to_vec(); + message.push(0x80); + while message.len() % 64 != 56 { + message.push(0); + } + message.extend_from_slice(&((data.len() as u64) * 8).to_be_bytes()); + for block in message.chunks_exact(64) { + let mut w = [0u32; 64]; + for (i, word) in block.chunks_exact(4).enumerate() { + w[i] = u32::from_be_bytes([word[0], word[1], word[2], word[3]]); + } + for i in 16..64 { + let s0 = w[i - 15].rotate_right(7) ^ w[i - 15].rotate_right(18) ^ (w[i - 15] >> 3); + let s1 = w[i - 2].rotate_right(17) ^ w[i - 2].rotate_right(19) ^ (w[i - 2] >> 10); + w[i] = w[i - 16] + .wrapping_add(s0) + .wrapping_add(w[i - 7]) + .wrapping_add(s1); + } + let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut h] = state; + for i in 0..64 { + let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); + let ch = (e & f) ^ (!e & g); + let t1 = h + .wrapping_add(s1) + .wrapping_add(ch) + .wrapping_add(K[i]) + .wrapping_add(w[i]); + let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); + let maj = (a & b) ^ (a & c) ^ (b & c); + let t2 = s0.wrapping_add(maj); + h = g; + g = f; + f = e; + e = d.wrapping_add(t1); + d = c; + c = b; + b = a; + a = t1.wrapping_add(t2); + } + for (slot, value) in state.iter_mut().zip([a, b, c, d, e, f, g, h]) { + *slot = slot.wrapping_add(value); + } + } + let mut out = [0u8; 32]; + for (chunk, word) in out.chunks_exact_mut(4).zip(state) { + chunk.copy_from_slice(&word.to_be_bytes()); + } + out + } +} diff --git a/rust/tests/harness_infra.rs b/rust/tests/harness_infra.rs new file mode 100644 index 00000000..c9ba9ae3 --- /dev/null +++ b/rust/tests/harness_infra.rs @@ -0,0 +1,1039 @@ +//! Controls for the replay infrastructure itself: the coordinator transport, +//! strict JSON, schema validation, inventory ids and the witness-evidence check. +//! These port the harness tests of `go/replay_coordinator_test.go`, +//! `go/behavior_replay_test.go` and `go/witness_evidence_test.go`. +//! +//! The transport tests spawn `node`; `node` on `PATH` must be Node 24 (see +//! `formal::transport`). Prepend `/opt/homebrew/opt/node@24/bin` to `PATH` +//! when the default toolchain is another major version. + +mod formal; + +use formal::inventory::{ + percent_encode_component, profile_registry_check_text, protocol_case_id, registry_check, + regression_paths, repo_root, require_behavior_profile, scenario_case_id, trace_case_id, + trace_kind, witness_case_id, BEHAVIOR_PROFILE_VERSIONS, +}; +use formal::json::{decode_itf, json_equal, sorted_keys, strict_parse}; +use formal::report::Report; +use formal::schema::{observation_error, Schema}; +use formal::transport::{read_frame, Coordinator, Prepared, SETTLEMENT}; +use formal::witness::{ + check_witness_evidence, file_sha256, sha256_hex, Digest, Evidence, Label, Trace, +}; +use serde_json::{json, Map, Value}; +use std::collections::HashMap; +use std::io::BufReader; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +fn node(program: &str) -> Command { + let mut command = Command::new("node"); + command.arg("-e").arg(program); + command +} + +fn request(op: &str) -> Map { + let mut request = Map::new(); + request.insert("op".to_string(), Value::from(op)); + request +} + +/// The flat all-zero record `$defs/coreObservation` accepts. +fn healthy_core_observation() -> Value { + let mut observed = Map::new(); + for field in "sourceVersion lastResult outsideLoaderCalls requestLoaderCalls localLoaderCalls coalescedLoaderCalls remoteLoaderCalls redisReads redisWrites".split(' ') { + observed.insert(field.to_string(), Value::from(0)); + } + Value::Object(observed) +} + +/// The empty behavior record; `events` is present exactly when the fixture observes. +fn empty_behavior_observation(observe: bool) -> Value { + let mut observed = Map::new(); + for field in [ + "loaders", + "reads", + "writes", + "invalidations", + "loads", + "dumps", + "policyCalls", + "classifications", + "comparisons", + ] { + observed.insert(field.to_string(), Value::from(0)); + } + for field in [ + "calls", + "maintenance", + "sourceScopes", + "writeTtls", + "shadow", + "recovery", + ] { + observed.insert(field.to_string(), Value::Array(Vec::new())); + } + if observe { + observed.insert("events".to_string(), Value::Array(Vec::new())); + } + Value::Object(observed) +} + +fn with(base: &Value, field: &str, value: Value) -> Value { + let mut changed = base.as_object().cloned().unwrap_or_default(); + changed.insert(field.to_string(), value); + Value::Object(changed) +} + +fn without(base: &Value, field: &str) -> Value { + let mut changed = base.as_object().cloned().unwrap_or_default(); + changed.remove(field); + Value::Object(changed) +} + +fn prepared_control() -> Prepared { + Prepared { + session: "1".to_string(), + steps: 2, + observation: "coreObservation".to_string(), + fixture: Value::Object(Map::new()), + setup: Vec::new(), + actions: vec!["init".to_string(), "outsideCall".to_string()], + } +} + +struct TempDir(PathBuf); + +impl TempDir { + fn new(name: &str) -> TempDir { + let path = std::env::temp_dir().join(format!( + "dialcache-rust-harness-{}-{name}", + std::process::id() + )); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir_all(&path).expect("temp dir"); + TempDir(path) + } + + fn write(&self, relative: &str, content: &str) { + let path = self.0.join(relative); + std::fs::create_dir_all(path.parent().expect("parent")).expect("mkdir"); + std::fs::write(&path, content).expect("write"); + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[test] +fn coordinator_answers_repeated_profile_requests() { + let mut coordinator = Coordinator::spawn().expect("shared replay requires Node 24 on PATH"); + let mut profiles = Map::new(); + for _ in 0..20 { + let result = coordinator.call(request("profiles")).expect("profiles"); + assert_eq!( + result.get("settlement").and_then(Value::as_str), + Some(SETTLEMENT) + ); + profiles = result + .get("profiles") + .and_then(Value::as_object) + .cloned() + .expect("profiles object"); + } + assert_eq!(profiles.len(), 15, "profiles: {}", sorted_keys(&profiles)); + for name in [ + "core", + "effects", + "local-clock", + "scope", + "policy", + "shadow", + "recovery", + ] { + assert!(profiles.contains_key(name), "missing profile {name}"); + } + for (name, actions) in &profiles { + let actions = actions + .as_array() + .unwrap_or_else(|| panic!("{name} actions must be a list")); + assert!(!actions.is_empty(), "{name} lists no actions"); + assert!( + actions + .iter() + .all(|action| action.as_str().is_some_and(|action| !action.is_empty())), + "{name} has a malformed action" + ); + } + coordinator.finish().expect("coordinator exits cleanly"); +} + +#[test] +fn coordinator_rejects_broken_responses() { + let cases = [ + ("malformed JSON", "not-json"), + ( + "unknown version", + r#"{"version":2,"id":1,"ok":true,"result":{}}"#, + ), + ( + "wrong sequence", + r#"{"version":1,"id":2,"ok":true,"result":{}}"#, + ), + ( + "unknown member", + r#"{"version":1,"id":1,"ok":true,"result":{},"extra":true}"#, + ), + ( + "duplicate member", + r#"{"version":1,"id":1,"ok":true,"result":{},"result":{}}"#, + ), + ( + "missing status", + r#"{"version":1,"id":1,"error":"missing status"}"#, + ), + ( + "mixed success", + r#"{"version":1,"id":1,"ok":true,"result":{},"error":"hidden"}"#, + ), + ]; + for (name, response) in cases { + let encoded = serde_json::to_string(&format!("{response}\n")).expect("encode"); + let program = format!("process.stdin.once('data', () => {{ process.stdout.write({encoded}); process.stdin.resume(); }});"); + let mut coordinator = + Coordinator::start(node(&program), Duration::from_secs(1), false).expect("start"); + assert!( + coordinator.call(request("profiles")).is_err(), + "{name}: accepted malformed coordinator response" + ); + coordinator + .finish() + .unwrap_or_else(|error| panic!("{name}: {error}")); + } +} + +#[test] +fn coordinator_times_out_and_closes_broken_process() { + // Both a blocked RPC and a child ignoring EOF must be bounded by real time. + let started = Instant::now(); + let mut coordinator = Coordinator::start( + node("process.stdin.resume(); setInterval(() => {}, 1000);"), + Duration::from_millis(100), + true, + ) + .expect("start"); + let error = coordinator + .call(request("profiles")) + .expect_err("accepted a process that never acknowledged the request"); + assert!( + error.contains("coordinator unavailable or exceeded real-time request limit"), + "{error}" + ); + coordinator + .finish() + .expect("expected failure is not reported"); + assert!( + started.elapsed() < Duration::from_secs(10), + "watchdog did not bound the broken process" + ); +} + +#[test] +fn coordinator_rejects_premature_completion_and_empty_commands() { + let cases = [ + ("premature completion", r#"{"complete":true,"steps":2}"#), + ( + "empty commands", + r#"{"complete":false,"index":1,"inputs":[]}"#, + ), + ( + "fractional index", + r#"{"complete":false,"index":1.5,"inputs":[{"op":"begin"}]}"#, + ), + ( + "non-command input", + r#"{"complete":false,"index":1,"inputs":[null]}"#, + ), + ( + "missing argument", + r#"{"complete":false,"index":1,"inputs":[{"op":"resolve"}]}"#, + ), + ( + "wrong argument type", + r#"{"complete":false,"index":1,"inputs":[{"op":"resolve","loader":"0"}]}"#, + ), + ( + "unexpected argument", + r#"{"complete":false,"index":1,"inputs":[{"op":"begin","expected":0}]}"#, + ), + ( + "unknown operation", + r#"{"complete":false,"index":1,"inputs":[{"op":"invented"}]}"#, + ), + ]; + for (name, result) in cases { + let program = format!( + "require('readline').createInterface({{input:process.stdin}}).on('line', line => {{ const request=JSON.parse(line); process.stdout.write(JSON.stringify({{version:1,id:request.id,ok:true,result:{result}}})+'\\n'); }});" + ); + let mut coordinator = + Coordinator::start(node(&program), Duration::from_secs(1), false).expect("start"); + let mut applied = 0; + // A well-formed core observation passes the local shape check, so the + // coordinator's malformed reply is what execution must reject. + let error = coordinator.execute( + &prepared_control(), + &mut |_input| { + applied += 1; + Ok(()) + }, + &mut healthy_core_observation, + &mut || 0, + &mut [], + ); + assert!( + error.is_err() && applied == 0, + "{name}: malformed reply advanced execution: {error:?}, commands={applied}" + ); + coordinator + .finish() + .unwrap_or_else(|error| panic!("{name}: {error}")); + } +} + +#[test] +fn transport_validates_observations_locally() { + let schema = Schema::load().expect("schema"); + let defs = schema.defs(); + let behavior = empty_behavior_observation(true); + let local = empty_behavior_observation(false); + for (definition, observed) in [ + ("behaviorObservation", &behavior), + ("coreObservation", &healthy_core_observation()), + ("localClockObservation", &local), + ] { + observation_error(observed, definition, defs) + .unwrap_or_else(|error| panic!("well-formed {definition} rejected: {error}")); + } + let core = healthy_core_observation(); + let controls: Vec<(&str, &str, Value)> = vec![ + ( + "string counter", + "behaviorObservation", + with(&behavior, "loaders", json!("1")), + ), + ( + "pending call value", + "behaviorObservation", + with(&behavior, "calls", json!([{"status": "value"}])), + ), + ( + "invented event", + "behaviorObservation", + with(&behavior, "events", json!([{"event": "invented"}])), + ), + ( + "unknown field", + "behaviorObservation", + with(&behavior, "extra", json!(1)), + ), + ( + "missing list", + "behaviorObservation", + without(&behavior, "maintenance"), + ), + ( + "negative counter", + "coreObservation", + with(&core, "redisReads", json!(-1)), + ), + ( + "missing counter", + "coreObservation", + without(&core, "redisWrites"), + ), + ( + "fractional counter", + "coreObservation", + with(&core, "redisReads", json!(0.5)), + ), + ( + "structured local call", + "localClockObservation", + with(&local, "calls", json!([{"status": "pending"}])), + ), + ( + "events on local clock", + "localClockObservation", + with(&local, "events", json!([])), + ), + ("non-object observation", "behaviorObservation", json!([])), + ("nil observation", "coreObservation", Value::Null), + ]; + for (name, definition, observed) in &controls { + let error = observation_error(observed, definition, defs) + .err() + .unwrap_or_else(|| panic!("{name}: malformed {definition} accepted: {observed}")); + assert!( + error.starts_with(&format!( + "driver produced a malformed {definition} observation: " + )), + "{name}: shape defect lacks the driver attribution: {error}" + ); + assert!( + !carries_comparison_markers(&error), + "{name}: shape defect acquired comparison markers: {error}" + ); + assert!( + !error.contains("\"1\"") && !error.contains("invented"), + "{name}: diagnostic leaked a value: {error}" + ); + } + for definition in ["", "invented", "observedEvent", "command"] { + let error = observation_error(&behavior, definition, defs) + .err() + .unwrap_or_else(|| panic!("definition {definition:?} accepted")); + assert!( + error.contains("unknown replay observation definition"), + "{definition:?}: {error}" + ); + } + + // Rejected before any request reaches the coordinator: the stand-in exits + // nonzero if an observe request ever arrives, which finish() reports. + let program = "require('readline').createInterface({input:process.stdin}).on('line', line => { const request=JSON.parse(line); if (request.op === 'observe') process.exit(3); process.stdout.write(JSON.stringify({version:1,id:request.id,ok:true,result:{complete:true,steps:2}})+'\\n'); });"; + let mut coordinator = + Coordinator::start(node(program), Duration::from_secs(1), false).expect("start"); + let mut applied = 0; + let mut prepared = prepared_control(); + prepared.setup = vec![json!({"op": "bumpSource"})]; + let error = coordinator + .execute( + &prepared, + &mut |_input| { + applied += 1; + Ok(()) + }, + &mut || with(&healthy_core_observation(), "redisReads", json!("many")), + &mut || 0, + &mut [], + ) + .expect_err("malformed observation accepted"); + assert!( + error.contains("driver produced a malformed coreObservation observation"), + "not attributed to the driver: {error}" + ); + assert_eq!(applied, 1, "setup must run before the first observation"); + coordinator + .finish() + .expect("an observe request reached the coordinator"); + + // prepare rejects an unknown observation definition. + let program = "require('readline').createInterface({input:process.stdin}).on('line', line => { const request=JSON.parse(line); process.stdout.write(JSON.stringify({version:1,id:request.id,ok:true,result:{session:'1',settlement:'causally-ready-v1',observation:'observedEvent',fixture:{},setup:[],actions:['init','outsideCall'],steps:2}})+'\\n'); });"; + let mut coordinator = + Coordinator::start(node(program), Duration::from_secs(1), false).expect("start"); + let error = coordinator + .prepare("core", Path::new("control.itf.json"), None) + .expect_err("unknown observation definition accepted"); + assert!( + error.contains("malformed replay observation definition"), + "{error}" + ); + coordinator.finish().expect("clean exit"); +} + +fn carries_comparison_markers(text: &str) -> bool { + text.find("expected:") + .is_some_and(|at| text[at..].contains("actual:")) +} + +#[test] +fn coordinator_prepares_and_replays_the_smoke_history_shape() { + // The real coordinator's prepare result must satisfy every structural check. + let mut coordinator = Coordinator::spawn().expect("shared replay requires Node 24 on PATH"); + let path = repo_root().join("formal/conformance-smoke.itf.json"); + let raw = std::fs::read_to_string(&path).expect("smoke history"); + let prepared = coordinator + .prepare("core", &path, Some(&raw)) + .expect("prepare"); + assert_eq!(prepared.observation, "coreObservation"); + assert_eq!(prepared.actions.len() as i64, prepared.steps); + assert_eq!(prepared.actions[0], "init"); + assert!(prepared.fixture.is_object()); + // Discarding is the early-exit path execute() takes; a stale session is an error. + let mut discard = request("discard"); + discard.insert("session".to_string(), Value::from(prepared.session.clone())); + coordinator.call(discard.clone()).expect("discard"); + let error = coordinator + .call(discard) + .expect_err("second discard accepted"); + assert!(error.starts_with("shared replay: "), "{error}"); + let error = coordinator + .prepare("core", Path::new("missing-history"), Some("{")) + .expect_err("malformed history accepted"); + assert!(error.starts_with("shared replay: "), "{error}"); + coordinator.finish().expect("clean exit"); +} + +#[test] +fn frames_are_bounded_before_buffering() { + let mut reader = + BufReader::with_capacity(16, "12345678901234567890123456789012345\n".as_bytes()); + let error = read_frame(&mut reader, 32).expect_err("oversized frame accepted"); + assert!(error.contains("oversized"), "{error}"); + let mut reader = BufReader::with_capacity(16, "{\"value\":1}".as_bytes()); + let error = read_frame(&mut reader, 32).expect_err("unterminated frame accepted"); + assert!(!error.contains("oversized"), "{error}"); + let mut reader = BufReader::with_capacity(4, "{\"value\":1}\nnext\n".as_bytes()); + assert_eq!( + read_frame(&mut reader, 32).expect("frame"), + b"{\"value\":1}\n" + ); + assert_eq!(read_frame(&mut reader, 32).expect("frame"), b"next\n"); +} + +#[test] +fn strict_json_rejects_ambiguous_inputs() { + for raw in [ + r#"{"action":1,"action":2}"#, + r#"{"action":1,"action":2}"#, + r#"[{"s":{"calls":[],"calls":[]}}]"#, + r#"{"value":NaN}"#, + r#"{"value":1} trailing"#, + "{", + "1e999", + ] { + assert!( + strict_parse(raw).is_err(), + "ambiguous/malformed JSON accepted: {raw}" + ); + } + for raw in [ + r#"{"value":"escaped \\\" delimiter } ]","nested":[null,false,1,-2.5e3,{"empty":{}}]}"#, + r#"{"a":1,"b":2}"#, + r#"[{},[],true,false,null,"",1]"#, + " {\"a\" : { \"b\" : [ 1 , 2 ] } , \"c\" : \"x\" }\n", + ] { + strict_parse(raw).unwrap_or_else(|error| panic!("valid JSON rejected: {raw}: {error}")); + } +} + +#[test] +fn json_projection_distinguishes_values() { + assert!( + json_equal( + &json!({"n": 1, "a": [false, null, ""]}), + &json!({"n": 1.0, "a": [false, null, ""]}) + ), + "equivalent JSON numbers differed" + ); + for (a, b) in [ + (json!(false), json!(0)), + (Value::Null, json!(false)), + (json!("1"), json!(1)), + (json!([]), Value::Null), + (json!({"x": null}), json!({})), + ] { + assert!( + !json_equal(&a, &b), + "distinct observations compared equal: {a} {b}" + ); + } +} + +#[test] +fn itf_integers_decode_exactly_or_fail() { + let decoded = + decode_itf(json!({"s": {"calls": [{"#bigint": "1"}, {"#bigint": "-7"}]}, "n": 2.5})) + .expect("decode"); + assert!( + json_equal(&decoded, &json!({"s": {"calls": [1, -7]}, "n": 2.5})), + "{decoded}" + ); + for (raw, message) in [ + (json!({"#bigint": "9007199254740992"}), "unsafe ITF integer"), + ( + json!({"#bigint": "-9007199254740992"}), + "unsafe ITF integer", + ), + (json!({"#bigint": "1.5"}), "unsafe ITF integer"), + (json!({"#bigint": 1}), "unsafe ITF integer"), + ( + json!({"#bigint": "1", "extra": true}), + "malformed ITF integer", + ), + (json!([9007199254740992i64]), "unsafe JSON number"), + ] { + let error = decode_itf(raw.clone()) + .err() + .unwrap_or_else(|| panic!("accepted {raw}")); + assert_eq!(error, message, "{raw}"); + } + assert!(json_equal( + &decode_itf(json!({"#bigint": "9007199254740991"})).expect("boundary"), + &json!(9007199254740991i64) + )); +} + +#[test] +fn inventory_ids_follow_the_shared_scheme() { + assert_eq!(percent_encode_component("a b"), "a%20b"); + assert_eq!( + percent_encode_component("scope: nested/close"), + "scope%3A%20nested%2Fclose" + ); + assert_eq!(percent_encode_component("café ✓"), "caf%C3%A9%20%E2%9C%93"); + assert_eq!( + percent_encode_component("A-Z_a.z!~*'()09"), + "A-Z_a.z!~*'()09" + ); + assert_eq!(percent_encode_component("#?&=+"), "%23%3F%26%3D%2B"); + assert_eq!( + scenario_case_id("stale recovery", "adapter legacy null"), + "scenario/stale%20recovery/adapter%20legacy%20null" + ); + assert_eq!( + protocol_case_id("keyVectors", "user id: 1"), + "protocol/keyVectors/user%20id%3A%201" + ); + assert_eq!(witness_case_id("effects"), "witness/effects"); + + let sampled = Path::new(".formal-traces/features/scope/trace_12.itf.json"); + assert_eq!(trace_kind(sampled), "sampled"); + assert_eq!(trace_case_id("scope", sampled, true), "sampled/scope/12"); + assert_eq!( + trace_case_id("scope", sampled, false), + "smoke/scope/trace_12.itf.json" + ); + let regression = Path::new( + ".formal-traces/features/../regressions/scope/absentValueIsMemoizedTest.itf.json", + ); + assert_eq!(trace_kind(regression), "regression"); + assert_eq!( + trace_case_id("scope", regression, true), + "regression/scope/absentValueIsMemoizedTest" + ); + assert_eq!( + trace_case_id("scope", regression, false), + "regression/scope/absentValueIsMemoizedTest" + ); + let smoke = repo_root().join("formal/scope-smoke.itf.json"); + assert_eq!( + trace_case_id("scope", &smoke, false), + "smoke/scope/scope-smoke.itf.json" + ); + assert_eq!( + trace_case_id("core", Path::new("/tmp/custom.itf.json"), true), + "smoke/core/custom.itf.json" + ); + assert_eq!( + trace_case_id("core", Path::new("/x/conformance/trace_007.itf.json"), true), + "smoke/core/trace_007.itf.json" + ); +} + +#[test] +fn missing_scheduled_regressions_fail_corpus_selection() { + let corpus = TempDir::new("effects-corpus"); + let error = + regression_paths("effects", &corpus.0).expect_err("missing scheduled histories accepted"); + assert!(error.contains("missing Quint regression"), "{error}"); + assert!(regression_paths("not-a-profile", &corpus.0) + .expect("no regressions") + .is_empty()); +} + +#[test] +fn registry_checks_match_go() { + registry_check().expect("core registry"); + for (profile, _) in BEHAVIOR_PROFILE_VERSIONS { + require_behavior_profile(profile).unwrap_or_else(|error| panic!("{profile}: {error}")); + } + assert!( + require_behavior_profile("core").is_err(), + "core is not a behavior profile" + ); + let raw = std::fs::read_to_string(repo_root().join("formal/profiles.json")).expect("registry"); + for mode in ["version", "missing", "duplicate", "model", "schema"] { + let mut registry = strict_parse(&raw).expect("parse"); + let profiles = registry["profiles"].as_array().cloned().expect("profiles"); + let at = profiles + .iter() + .position(|profile| profile["id"] == "scope") + .expect("scope profile"); + let mut profile = profiles[at].clone(); + match mode { + "version" => profile["version"] = json!(999), + "model" => profile["model"] = json!("formal/unknown.qnt"), + "schema" => registry["behavioralSchemaVersion"] = json!(999), + _ => {} + } + let mut changed = profiles.clone(); + match mode { + "missing" => { + changed.remove(at); + } + "duplicate" => changed.push(profile.clone()), + _ => changed[at] = profile, + } + registry["profiles"] = Value::Array(changed); + assert!( + profile_registry_check_text(®istry.to_string(), "scope", 2).is_err(), + "{mode}: unsupported profile registry accepted" + ); + } +} + +#[test] +fn report_writes_jsonl_records() { + let directory = TempDir::new("report"); + let path = directory.0.join("rust-replay.jsonl"); + let mut report = Report::create(Some(&path)).expect("create"); + report.start().expect("start"); + report + .case("sampled/core/0", &Ok(()), 10, 20) + .expect("case"); + report + .case( + "witness/scope", + &Err("stale witness definition x".to_string()), + 30, + 40, + ) + .expect("case"); + let summary = report.finish().expect("finish"); + assert_eq!( + (summary.cases, summary.failed, summary.status()), + (2, 1, "failed") + ); + let text = std::fs::read_to_string(&path).expect("report"); + let records: Vec = text + .lines() + .map(|line| strict_parse(line).expect("record")) + .collect(); + assert_eq!(records.len(), 4); + assert_eq!(records[0]["kind"], "start"); + assert_eq!(records[0]["schemaVersion"], 1); + assert_eq!(records[0]["implementation"], "rust"); + assert!(records[0]["startedAt"] + .as_i64() + .is_some_and(|ms| ms > 1_700_000_000_000)); + assert_eq!( + records[1], + json!({"kind": "case", "id": "sampled/core/0", "status": "passed", "startedAt": 10, "finishedAt": 20}) + ); + assert_eq!( + records[2], + json!({"kind": "case", "id": "witness/scope", "status": "failed", "startedAt": 30, "finishedAt": 40, "message": "stale witness definition x"}) + ); + assert_eq!(records[3]["kind"], "finish"); + assert_eq!(records[3]["status"], "failed"); + assert_eq!(records[3]["cases"], 2); + assert_eq!(records[3]["failed"], 1); + let mut silent = Report::create(None).expect("no-op"); + silent.start().expect("start"); + assert!(silent.run_case("x", || Ok(())).is_ok()); + assert_eq!(silent.finish().expect("finish").cases, 1); +} + +#[test] +fn sha256_matches_known_vectors() { + assert_eq!( + sha256_hex(b""), + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ); + assert_eq!( + sha256_hex(b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + assert_eq!( + sha256_hex(b"abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq"), + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1" + ); + assert_eq!( + sha256_hex(&[b'a'; 1_000_000]), + "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0" + ); + // 55, 56 and 64 byte messages straddle the padding boundary. + assert_eq!( + sha256_hex(&[0u8; 55]), + "02779466cdec163811d078815c633f21901413081449002f24aa3e80f0b88ef7" + ); + assert_eq!( + sha256_hex(&[0u8; 56]), + "d4817aa5497628e7c77e6b606107042bbba3130888c5f47a375e6179be789fbb" + ); + assert_eq!( + sha256_hex(&[0u8; 64]), + "f5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b" + ); + let schema = repo_root().join("formal/replay/protocol.schema.json"); + let expected = Command::new("shasum") + .arg("-a") + .arg("256") + .arg(&schema) + .output() + .ok() + .filter(|output| output.status.success()); + if let Some(output) = expected { + let text = String::from_utf8_lossy(&output.stdout); + assert_eq!( + text.split_whitespace().next(), + Some(file_sha256(&schema).expect("hash").as_str()) + ); + } +} + +#[test] +fn witness_evidence_binds_shared_replay_sources() { + let root = TempDir::new("witness-root"); + let inputs = [ + "formal/profiles.json", + "formal/coverage-witnesses.json", + "formal/execution.json", + "formal/dialcache-effects-conformance.qnt", + "formal/conformance-observations.qnt", + "formal/replay/coordinator.mjs", + "formal/replay/mapping.mjs", + ]; + for path in inputs { + root.write(path, "reviewed input"); + } + root.write("formal/profiles.json", r#"{"profiles":[{"id":"effects"}],"replaySources":["formal/replay/coordinator.mjs","formal/replay/mapping.mjs"]}"#); + root.write( + "formal/coverage-witnesses.json", + r#"{"effects":["observed"]}"#, + ); + root.write("formal/execution.json", r#"{"libraries":[]}"#); + root.write("trace.itf.json", "controlled trace"); + let trace = root.0.join("trace.itf.json"); + let cite = |name: &str, kind: &str, checkpoints: Vec| Trace { + name: name.to_string(), + kind: kind.to_string(), + checkpoints, + }; + let mut evidence = Evidence { + schema_version: 2, + profile: "effects".to_string(), + traces: 1, + required: vec!["observed".to_string()], + seen: vec!["observed".to_string()], + labels: Some(HashMap::from([( + "observed".to_string(), + Label { + sampled: 1, + regression: 0, + traces: vec![cite("trace.itf.json", "sampled", vec![1])], + }, + )])), + inputs: Vec::new(), + corpus: Vec::new(), + }; + for path in inputs { + evidence.inputs.push(Digest { + path: path.to_string(), + name: String::new(), + sha256: file_sha256(&root.0.join(path)).expect("hash"), + }); + } + evidence.corpus = vec![Digest { + path: String::new(), + name: "trace.itf.json".to_string(), + sha256: file_sha256(&trace).expect("hash"), + }]; + let write = |evidence: &Evidence| { + root.write( + "effects.json", + &serde_json::to_string(evidence).expect("encode"), + ) + }; + write(&evidence); + let check = + || check_witness_evidence(&root.0, "effects", &root.0, std::slice::from_ref(&trace)); + check().expect("valid evidence rejected"); + + let expect = |patch: &dyn Fn(&mut Evidence), fragment: &str| { + let mut copied = evidence.clone(); + patch(&mut copied); + write(&copied); + let error = check() + .err() + .unwrap_or_else(|| panic!("evidence accepted; expected {fragment:?}")); + assert!(error.contains(fragment), "expected {fragment:?} in {error}"); + }; + expect(&|e| e.schema_version = 1, "unsupported"); + expect(&|e| e.labels = Some(HashMap::new()), "lacks provenance"); + expect(&|e| e.labels = None, "lacks per-label provenance"); + expect( + &|e| { + e.labels = Some(HashMap::from([( + "observed".to_string(), + Label { + sampled: 1, + regression: 0, + traces: vec![cite("other.itf.json", "sampled", vec![1])], + }, + )])) + }, + "unknown history", + ); + expect( + &|e| { + e.labels = Some(HashMap::from([( + "observed".to_string(), + Label { + sampled: 0, + regression: 1, + traces: vec![cite("trace.itf.json", "regression", vec![1])], + }, + )])) + }, + "as regression", + ); + expect( + &|e| { + e.labels = Some(HashMap::from([( + "observed".to_string(), + Label { + sampled: 2, + regression: 0, + traces: vec![cite("trace.itf.json", "sampled", vec![1])], + }, + )])) + }, + "cites 1 and 0", + ); + expect( + &|e| { + e.labels = Some(HashMap::from([( + "observed".to_string(), + Label { + sampled: 1, + regression: 0, + traces: vec![cite("trace.itf.json", "sampled", Vec::new())], + }, + )])) + }, + "without a checkpoint", + ); + expect( + &|e| e.seen = vec!["observed".to_string(), "observed".to_string()], + "duplicate witness observed", + ); + expect(&|e| e.seen = Vec::new(), "missing witness observed"); + expect( + &|e| e.required = Vec::new(), + "required witness registry differs", + ); + expect( + &|e| e.inputs.pop().map(drop).unwrap_or_default(), + "incomplete witness definition fingerprints", + ); + expect( + &|e| e.inputs[6].path = "formal/replay/other.mjs".to_string(), + "unexpected witness input formal/replay/other.mjs", + ); + expect( + &|e| e.corpus[0].sha256 = "00".to_string(), + "witness corpus differs at trace.itf.json", + ); + expect( + &|e| e.corpus[0].name = "other.itf.json".to_string(), + "witness corpus differs at other.itf.json", + ); + + // Go-encoded evidence spells empty lists as null; the reader accepts that. + root.write( + "effects.json", + &serde_json::to_string(&evidence) + .expect("encode") + .replace(r#""regression":0"#, r#""regression":0,"extra":null"#), + ); + check().expect("unknown members are ignored"); + write(&evidence); + check().expect("restored evidence rejected"); + + root.write("formal/replay/mapping.mjs", "changed input mapping"); + let error = check().expect_err("changed shared mapping accepted"); + assert!( + error.contains("stale witness definition formal/replay/mapping.mjs"), + "{error}" + ); + root.write("formal/replay/mapping.mjs", "reviewed input"); + root.write("formal/replay/new-helper.mjs", "unregistered helper"); + let error = check().expect_err("new dependency accepted"); + assert!(error.contains("inventory differs"), "{error}"); + std::fs::remove_file(root.0.join("formal/replay/new-helper.mjs")).expect("remove"); + std::fs::remove_file(root.0.join("formal/replay/mapping.mjs")).expect("remove"); + let error = check().expect_err("missing dependency accepted"); + assert!(error.contains("inventory differs"), "{error}"); +} + +#[test] +fn observe_requests_reach_the_coordinator_comparison() { + // A driver that reports zeros diverges from the smoke history, but only + // after the coordinator accepted the observe request shape and compared the + // record: the failure is an observation mismatch, not a protocol violation. + let mut coordinator = Coordinator::spawn().expect("shared replay requires Node 24 on PATH"); + let path = repo_root().join("formal/conformance-smoke.itf.json"); + let prepared = coordinator.prepare("core", &path, None).expect("prepare"); + let mut applied = Vec::new(); + let error = coordinator + .execute( + &prepared, + &mut |input| { + applied.push(input.clone()); + Ok(()) + }, + &mut healthy_core_observation, + &mut || 1_788_868_800_000, + &mut [], + ) + .expect_err("a zero observation matched the smoke history"); + assert!( + error.starts_with("shared replay: ") && error.contains("Observation mismatch"), + "{error}" + ); + assert!( + !error.contains("Malformed") && !error.contains("schema"), + "observe request was rejected structurally: {error}" + ); + assert_eq!( + applied.len(), + prepared.setup.len(), + "commands applied before the first observation mismatch" + ); + // The failed session was discarded, so the coordinator no longer knows it. + let mut observe = request("observe"); + observe.insert("session".to_string(), Value::from(prepared.session.clone())); + observe.insert("index".to_string(), Value::from(0)); + observe.insert("settlement".to_string(), Value::from(SETTLEMENT)); + observe.insert("observed".to_string(), healthy_core_observation()); + observe.insert( + "environment".to_string(), + json!({"wallMs": 1_788_868_800_000i64}), + ); + let error = coordinator + .call(observe) + .expect_err("discarded session accepted"); + assert!(error.contains("Unknown replay session"), "{error}"); + coordinator.finish().expect("clean exit"); +} + +#[test] +fn checked_in_replay_sources_match_the_registry() { + // The real registry must agree with the real formal/replay tree, or no + // witness evidence produced from this checkout could ever be accepted. + let sources = + formal::witness::shared_replay_sources(&repo_root()).expect("shared replay sources"); + assert!(sources + .iter() + .any(|path| path == "formal/replay/coordinator.mjs")); + assert!(sources + .iter() + .any(|path| path == "formal/replay/protocol.schema.json")); +} From 53d8d1cadbace254a5e32686270d6bc620e3d0f5 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 00:31:17 -0700 Subject: [PATCH 08/31] refactor(rust): split timers into a Runtime trait, add scope guard, test-util runtime and codec adapter --- rust/Cargo.lock | 1 + rust/Cargo.toml | 3 + rust/src/clock.rs | 84 ++++------- rust/src/codec.rs | 36 +++-- rust/src/deadline.rs | 4 +- rust/src/engine.rs | 107 ++++++++++---- rust/src/execution.rs | 13 +- rust/src/flight.rs | 10 +- rust/src/lib.rs | 29 ++-- rust/src/observe.rs | 5 +- rust/src/runtime.rs | 40 +++++ rust/src/scope.rs | 6 + rust/src/shadow.rs | 13 +- rust/src/spawn.rs | 29 ---- rust/src/testing.rs | 330 ++++++++++++++++++++++++++++++++++++++++++ 15 files changed, 554 insertions(+), 156 deletions(-) create mode 100644 rust/src/runtime.rs delete mode 100644 rust/src/spawn.rs create mode 100644 rust/src/testing.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 08e728c3..e9381413 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -129,6 +129,7 @@ dependencies = [ name = "dialcache" version = "0.1.0" dependencies = [ + "dialcache", "futures", "hex", "log", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 2f68cc6d..039f909e 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -13,6 +13,8 @@ publish = false [features] default = ["tokio"] tokio = ["dep:tokio"] +# Deterministic clock and single-threaded runtime for tests. +test-util = [] redis = ["dep:redis", "dep:sha1", "dep:hex"] prometheus = ["dep:prometheus"] @@ -33,6 +35,7 @@ hex = { version = "0.4", optional = true } prometheus = { version = "0.14", optional = true } [dev-dependencies] +dialcache = { path = ".", features = ["test-util"] } tokio = { version = "1", features = ["full"] } hex = "0.4" serde_json = "1" diff --git a/rust/src/clock.rs b/rust/src/clock.rs index bbacccd7..9711e482 100644 --- a/rust/src/clock.rs +++ b/rust/src/clock.rs @@ -1,14 +1,12 @@ //! Wall and elapsed clocks. -use std::time::Duration; +use std::sync::OnceLock; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; -use futures::future::BoxFuture; - -/// Separates wall timestamps from monotonic elapsed time and timers. +/// Separates wall timestamps from monotonic elapsed time. /// /// Wall time stamps frames and invalidation markers. Elapsed time governs -/// deadlines and process-local expiry. Timers deliver deadline callbacks; a -/// controlled clock may deliver them on its own schedule. +/// deadlines and process-local expiry. Timers live on the [`Runtime`](crate::Runtime). pub trait Clock: Send + Sync + 'static { /// Epoch milliseconds from the application wall clock. fn wall_ms(&self) -> i64; @@ -21,62 +19,42 @@ pub trait Clock: Send + Sync + 'static { fn elapsed_ms(&self) -> i64 { self.elapsed().as_millis().min(i64::MAX as u128) as i64 } - /// A timer that completes once `duration` of this clock's time has passed. - fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()>; } -#[cfg(feature = "tokio")] -mod system { - use super::Clock; - use futures::future::BoxFuture; - use std::sync::OnceLock; - use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +static PROCESS_ORIGIN: OnceLock = OnceLock::new(); - static PROCESS_ORIGIN: OnceLock = OnceLock::new(); +/// The default clock: system wall time and a monotonic origin aligned to the +/// process-wide millisecond grid, so default instances share one local expiry grid. +#[derive(Debug, Clone)] +pub struct SystemClock { + origin: Instant, +} - /// The default clock: system wall time, a monotonic origin aligned to the - /// process-wide millisecond grid, and tokio timers. - #[derive(Debug, Clone)] - pub struct SystemClock { - origin: Instant, +impl SystemClock { + pub fn new() -> Self { + let process = *PROCESS_ORIGIN.get_or_init(Instant::now); + let now = Instant::now(); + let phase = now.saturating_duration_since(process).as_nanos() % 1_000_000; + let origin = now.checked_sub(Duration::from_nanos(phase as u64)).unwrap_or(now); + SystemClock { origin } } +} - impl SystemClock { - /// A clock whose whole-millisecond grid is shared by every default instance. - pub fn new() -> Self { - let process = *PROCESS_ORIGIN.get_or_init(Instant::now); - let now = Instant::now(); - let phase = now.saturating_duration_since(process).as_nanos() % 1_000_000; - let origin = now - .checked_sub(Duration::from_nanos(phase as u64)) - .unwrap_or(now); - SystemClock { origin } - } +impl Default for SystemClock { + fn default() -> Self { + Self::new() } +} - impl Default for SystemClock { - fn default() -> Self { - Self::new() - } +impl Clock for SystemClock { + fn wall_ms(&self) -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis().min(i64::MAX as u128) as i64) + .unwrap_or(0) } - impl Clock for SystemClock { - fn wall_ms(&self) -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis().min(i64::MAX as u128) as i64) - .unwrap_or(0) - } - - fn elapsed(&self) -> Duration { - self.origin.elapsed() - } - - fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { - Box::pin(tokio::time::sleep(duration)) - } + fn elapsed(&self) -> Duration { + self.origin.elapsed() } } - -#[cfg(feature = "tokio")] -pub use system::SystemClock; diff --git a/rust/src/codec.rs b/rust/src/codec.rs index 76b5a4dc..95875174 100644 --- a/rust/src/codec.rs +++ b/rust/src/codec.rs @@ -50,29 +50,33 @@ impl Payload { /// Encodes cached values to payloads and decodes them back. /// -/// Implement [`SyncCodec`] for ordinary serializers; it is adapted to this -/// asynchronous form automatically. Implement this trait directly only when -/// encoding or decoding must await. Every decode must return an independent -/// value: the cache may retain one payload and decode it more than once. +/// Encoding and decoding are asynchronous so a codec may await. For ordinary +/// synchronous serializers implement [`SyncCodec`] and wrap the codec in +/// [`FromSync`]. Every decode must return an independent value: the cache may +/// retain one payload and decode it more than once. pub trait Codec: Send + Sync + 'static { fn encode<'a>(&'a self, value: &'a T) -> BoxFuture<'a, Result>; fn decode(&self, payload: Payload) -> BoxFuture<'_, Result>; } -/// A synchronous codec. Blanket-adapted to [`Codec`]. +/// A synchronous codec. Wrap it in [`FromSync`] where a [`Codec`] is expected. pub trait SyncCodec: Send + Sync + 'static { fn encode(&self, value: &T) -> Result; fn decode(&self, payload: Payload) -> Result; } -impl> Codec for C { +/// Adapts a [`SyncCodec`] to [`Codec`]. +#[derive(Debug, Clone, Copy, Default)] +pub struct FromSync(pub C); + +impl> Codec for FromSync { fn encode<'a>(&'a self, value: &'a T) -> BoxFuture<'a, Result> { - let result = SyncCodec::encode(self, value); + let result = self.0.encode(value); Box::pin(std::future::ready(result)) } fn decode(&self, payload: Payload) -> BoxFuture<'_, Result> { - let result = SyncCodec::decode(self, payload); + let result = self.0.decode(payload); Box::pin(std::future::ready(result)) } } @@ -89,12 +93,12 @@ pub const JSON_UNDEFINED_SENTINEL: &str = "__dialcache_json_undefined_v1__"; #[derive(Debug, Clone, Copy, Default)] pub struct JsonCodec; -impl SyncCodec for JsonCodec { - fn encode(&self, value: &T) -> Result { +impl JsonCodec { + pub fn encode_value(value: &T) -> Result { Ok(Payload::text(serde_json::to_string(value)?)) } - fn decode(&self, payload: Payload) -> Result { + pub fn decode_value(payload: &Payload) -> Result { let text = payload.as_text(); if text == JSON_UNDEFINED_SENTINEL { return Ok(T::deserialize(serde_json::Value::Null)?); @@ -102,3 +106,13 @@ impl SyncCodec for J Ok(serde_json::from_str(&text)?) } } + +impl Codec for JsonCodec { + fn encode<'a>(&'a self, value: &'a T) -> BoxFuture<'a, Result> { + Box::pin(std::future::ready(JsonCodec::encode_value(value))) + } + + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result> { + Box::pin(std::future::ready(JsonCodec::decode_value(&payload))) + } +} diff --git a/rust/src/deadline.rs b/rust/src/deadline.rs index 445e1d7f..3f2f7eae 100644 --- a/rust/src/deadline.rs +++ b/rust/src/deadline.rs @@ -6,6 +6,7 @@ use futures::future::{select, Either}; use crate::clock::Clock; use crate::flight::Settled; +use crate::runtime::Runtime; /// Elapsed time since `start`, never negative. pub(crate) fn since(clock: &dyn Clock, start: Duration) -> Duration { @@ -21,6 +22,7 @@ pub(crate) fn seconds_since(clock: &dyn Clock, start: Duration) -> f64 { /// caller stops waiting. `None` budget waits without bound. pub(crate) async fn await_deadline( clock: &dyn Clock, + runtime: &dyn Runtime, pending: &Settled, started: Duration, budget_ms: Option, @@ -38,7 +40,7 @@ pub(crate) async fn await_deadline( break; } let remaining = budget.saturating_sub(since(clock, started)); - let timer = clock.sleep(remaining); + let timer = runtime.sleep(remaining); match select(pending.wait(), timer).await { Either::Left((value, _timer)) => { if since(clock, started) < budget { diff --git a/rust/src/engine.rs b/rust/src/engine.rs index 2ce3d5e5..e4572b1b 100644 --- a/rust/src/engine.rs +++ b/rust/src/engine.rs @@ -27,7 +27,7 @@ use crate::protocol::CompressionConfig; use crate::remote::{InvalidateRequest, Remote}; use crate::scope::{Owner, Scope}; use crate::shadow::ShadowFlight; -use crate::spawn::Spawner; +use crate::runtime::Runtime; /// Resolves a sparse runtime policy overlay once per enabled call. pub type PolicyProvider = Arc< @@ -56,7 +56,7 @@ pub(crate) struct Core { /// `None` disables new compressed writes; reads still decompress. pub(crate) compression: Option, pub(crate) clock: Arc, - pub(crate) spawner: Arc, + pub(crate) runtime: Arc, pub(crate) state: Mutex, } @@ -149,7 +149,7 @@ pub struct DialCacheBuilder { should_recover: Option, compression: Option>, clock: Option>, - spawner: Option>, + runtime: Option>, } impl std::fmt::Debug for DialCacheBuilder { @@ -175,7 +175,7 @@ impl DialCacheBuilder { should_recover: None, compression: None, clock: None, - spawner: None, + runtime: None, } } @@ -187,18 +187,35 @@ impl DialCacheBuilder { } /// The remote layer adapter. Without one, only request and local layers serve. - pub fn remote(mut self, remote: Arc) -> Self { + pub fn remote(mut self, remote: impl Remote) -> Self { + self.remote = Some(Arc::new(remote)); + self + } + + /// The remote layer adapter, shared. + pub fn remote_arc(mut self, remote: Arc) -> Self { self.remote = Some(remote); self } /// Receives every public diagnostic; also the shadow outcome hook. - pub fn observer(mut self, observer: Arc) -> Self { + pub fn observer(mut self, observer: impl Observer) -> Self { + self.observer = Some(Arc::new(observer)); + self + } + + /// Receives every public diagnostic, shared. + pub fn observer_arc(mut self, observer: Arc) -> Self { self.observer = Some(observer); self } - pub fn logger(mut self, logger: Arc) -> Self { + pub fn logger(mut self, logger: impl Logger) -> Self { + self.logger = Some(Arc::new(logger)); + self + } + + pub fn logger_arc(mut self, logger: Arc) -> Self { self.logger = Some(logger); self } @@ -262,13 +279,26 @@ impl DialCacheBuilder { self } - pub fn clock(mut self, clock: Arc) -> Self { + pub fn clock(mut self, clock: impl Clock) -> Self { + self.clock = Some(Arc::new(clock)); + self + } + + /// Share a clock between instances (for example a controlled test clock). + pub fn clock_arc(mut self, clock: Arc) -> Self { self.clock = Some(clock); self } - pub fn spawner(mut self, spawner: Arc) -> Self { - self.spawner = Some(spawner); + /// Share a runtime between instances. + pub fn runtime_arc(mut self, runtime: Arc) -> Self { + self.runtime = Some(runtime); + self + } + + /// Replace the executor and timer source. Defaults to tokio. + pub fn runtime(mut self, runtime: impl Runtime) -> Self { + self.runtime = Some(Arc::new(runtime)); self } @@ -321,9 +351,9 @@ impl DialCacheBuilder { Some(clock) => clock, None => default_clock()?, }; - let spawner = match self.spawner { - Some(spawner) => spawner, - None => default_spawner()?, + let runtime = match self.runtime { + Some(runtime) => runtime, + None => default_runtime()?, }; let core = Core { id: NEXT_CACHE_ID.fetch_add(1, Ordering::Relaxed), @@ -339,7 +369,7 @@ impl DialCacheBuilder { should_recover: self.should_recover, compression, clock, - spawner, + runtime, state: Mutex::new(CoreState { local, flights: HashMap::new(), @@ -353,24 +383,38 @@ impl DialCacheBuilder { } } -#[cfg(feature = "tokio")] fn default_clock() -> Result, ConfigError> { Ok(Arc::new(crate::clock::SystemClock::new())) } +#[cfg(feature = "tokio")] +fn default_runtime() -> Result, ConfigError> { + Ok(Arc::new(crate::runtime::TokioRuntime)) +} + #[cfg(not(feature = "tokio"))] -fn default_clock() -> Result, ConfigError> { - Err(ConfigError::invalid("DialCache needs a clock: enable the tokio feature or supply one with DialCacheBuilder::clock")) +fn default_runtime() -> Result, ConfigError> { + Err(ConfigError::invalid("DialCache needs a runtime: enable the tokio feature or supply one with DialCacheBuilder::runtime")) } -#[cfg(feature = "tokio")] -fn default_spawner() -> Result, ConfigError> { - Ok(Arc::new(crate::spawn::TokioSpawner)) +/// Holds the outermost enabled scope open until dropped. +#[derive(Debug)] +pub struct ScopeGuard { + scope: Scope, + owner: Arc, } -#[cfg(not(feature = "tokio"))] -fn default_spawner() -> Result, ConfigError> { - Err(ConfigError::invalid("DialCache needs a spawner: enable the tokio feature or supply one with DialCacheBuilder::spawner")) +impl ScopeGuard { + /// The enabled scope of this request. + pub fn scope(&self) -> &Scope { + &self.scope + } +} + +impl Drop for ScopeGuard { + fn drop(&mut self) { + self.owner.close(); + } } /// A cache instance: request, process-local and remote layers behind explicit enablement. @@ -399,6 +443,19 @@ impl DialCache { &self.core.namespace } + /// Open the outermost enabled scope and hold it in a guard. The scope + /// closes when the guard drops; retained [`Scope`] clones then pass through. + /// + /// ```ignore + /// let request = cache.enable_guard(); + /// let name = display_name.get(request.scope(), id).await?; + /// ``` + pub fn enable_guard(&self) -> ScopeGuard { + let owner = Owner::new(); + let scope = Scope { cache_id: self.core.id, owner: Some(owner.clone()), enabled: true }; + ScopeGuard { scope, owner } + } + /// Open the outermost enabled scope for `f`. The scope closes when the /// returned future completes; retained clones then pass through. pub async fn enable(&self, f: F) -> R @@ -552,7 +609,7 @@ impl DialCache { future_buffer_ms, }; let pending: Settled> = start_pending( - core.spawner.as_ref(), + core.runtime.as_ref(), async move { remote .invalidate(request) @@ -632,7 +689,7 @@ impl DialCache { Scope::outside() }; let pending: Settled = start_pending( - core.spawner.clone().as_ref(), + core.runtime.clone().as_ref(), crate::execution::run(core, scope, Arc::new(operation)), |message| Err(Error::Panic(message)), ); diff --git a/rust/src/execution.rs b/rust/src/execution.rs index f95f48aa..ece59371 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -122,7 +122,7 @@ pub(crate) async fn run(core: Arc, scope: Scope, op: Arc) let overlay: Result, BoxError> = match &core.provider { Some(provider) => { let pending = start_pending( - core.spawner.as_ref(), + core.runtime.as_ref(), { let provider = provider.clone(); let identity = identity.clone(); @@ -265,7 +265,7 @@ async fn source_with_budget( let start = clock.elapsed(); let budget = op.budget.millis(); let pending: Settled = start_pending( - core.spawner.as_ref(), + core.runtime.as_ref(), { let op = op.clone(); async move { call_load(&op, scope).await } @@ -273,7 +273,7 @@ async fn source_with_budget( |message| Err(Error::Panic(message)), ); let use_case = labels.use_case.to_string(); - let result = await_deadline(clock.as_ref(), &pending, start, budget, || { + let result = await_deadline(clock.as_ref(), core.runtime.as_ref(), &pending, start, budget, || { if let Some(flag) = timed_out { flag.store(true, Ordering::SeqCst); } @@ -629,9 +629,10 @@ impl Execution { }, }; let clock = self.core.clock.clone(); + let runtime = self.core.runtime.clone(); let start = clock.elapsed(); let raw: Settled = start_pending( - self.core.spawner.as_ref(), + self.core.runtime.as_ref(), async move { remote .read(request, context) @@ -644,7 +645,7 @@ impl Execution { let use_case = self.labels.use_case.to_string(); let waited = raw.clone(); let bounded = Box::pin(async move { - let result = await_deadline(clock.as_ref(), &waited, start, Some(timeout_ms), || { + let result = await_deadline(clock.as_ref(), runtime.as_ref(), &waited, start, Some(timeout_ms), || { cancel.cancel(); Err(Arc::new(RemoteReadTimeout { use_case, @@ -910,7 +911,7 @@ impl Execution { ttl_ms: ttl, }; let pending: Settled> = start_pending( - self.core.spawner.as_ref(), + self.core.runtime.as_ref(), async move { remote .write(request) diff --git a/rust/src/flight.rs b/rust/src/flight.rs index e7ff87b7..13730b91 100644 --- a/rust/src/flight.rs +++ b/rust/src/flight.rs @@ -13,7 +13,7 @@ use parking_lot::Mutex; use crate::error::Error; use crate::local::StoredValue; -use crate::spawn::Spawner; +use crate::runtime::Runtime; struct SettledState { value: Option, @@ -107,7 +107,7 @@ pub(crate) fn panic_message(payload: Box) -> Arc /// result, or with `on_panic` when it panics. Callers that stop waiting keep /// no ownership of the work. pub(crate) fn start_pending( - spawner: &dyn Spawner, + runtime: &dyn Runtime, work: F, on_panic: impl FnOnce(Arc) -> T + Send + 'static, ) -> Settled @@ -117,7 +117,7 @@ where { let settled = Settled::new(); let cell = settled.clone(); - spawner.spawn(Box::pin(async move { + runtime.spawn(Box::pin(async move { let outcome = match AssertUnwindSafe(work).catch_unwind().await { Ok(value) => value, Err(payload) => on_panic(panic_message(payload)), @@ -129,10 +129,10 @@ where /// Complete once work that is already runnable has progressed (the executor's /// deferred queue drained under a controlled scheduler). -pub(crate) async fn yield_deferred(spawner: &dyn Spawner) { +pub(crate) async fn yield_deferred(runtime: &dyn Runtime) { let done: Settled<()> = Settled::new(); let cell = done.clone(); - spawner.defer(Box::pin(async move { + runtime.defer(Box::pin(async move { cell.settle(()); })); done.wait().await; diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 17853add..f38b5ba0 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -26,33 +26,26 @@ pub mod policy; pub mod preview; pub mod protocol; pub mod remote; +pub mod runtime; mod scope; mod shadow; -pub mod spawn; +#[cfg(feature = "test-util")] +pub mod testing; mod use_case; pub use cancel::CancelToken; -pub use clock::Clock; -#[cfg(feature = "tokio")] -pub use clock::SystemClock; -pub use codec::{Codec, JsonCodec, Payload, SyncCodec}; -pub use engine::{ - CoalescingState, DialCache, DialCacheBuilder, PolicyProvider, ProcessCoalescingState, -}; +pub use clock::{Clock, SystemClock}; +pub use codec::{Codec, FromSync, JsonCodec, Payload, SyncCodec}; +pub use engine::{CoalescingState, DialCache, DialCacheBuilder, PolicyProvider, ProcessCoalescingState, ScopeGuard}; pub use error::{BoxError, ConfigError, Error, FallbackTimeout, RemoteReadTimeout, SharedError}; pub use identity::{normalize_args, ArgValue, Identity, IdentityError, Keys}; pub use local::{LocalEntry, LocalStore, LruLocalStore, StoredValue}; pub use observe::{Event, Labels, LogEvent, LogLevel, Logger, Observer, ShadowMismatchDetails}; pub use operation::{Comparator, Operation, Preview, RecoveryPredicate, SourceBudget}; -pub use policy::{ - Policy, PolicyDefaults, PolicyError, ResolvedPolicy, RuntimePolicy, ShadowPolicy, -}; -pub use remote::{ - Frame, InvalidateRequest, MissReason, ReadContext, ReadRequest, ReadResult, Remote, - WriteRequest, -}; -pub use scope::Scope; -pub use spawn::Spawner; +pub use policy::{Policy, PolicyDefaults, PolicyError, ResolvedPolicy, RuntimePolicy, ShadowPolicy}; +pub use remote::{Frame, InvalidateRequest, MissReason, ReadContext, ReadRequest, ReadResult, Remote, WriteRequest}; +pub use runtime::Runtime; #[cfg(feature = "tokio")] -pub use spawn::TokioSpawner; +pub use runtime::TokioRuntime; +pub use scope::Scope; pub use use_case::{KeySpec, UseCase, UseCaseBuilder}; diff --git a/rust/src/observe.rs b/rust/src/observe.rs index 8e4bafce..0140f42f 100644 --- a/rust/src/observe.rs +++ b/rust/src/observe.rs @@ -230,9 +230,10 @@ pub trait Observer: Send + Sync + 'static { /// Whether this observer consumes shadow validation outcomes. /// /// Shadow validation is diagnostic work that exists only to be observed, - /// so the cache admits a shadow job only when an observer reports `true`. + /// so the cache admits a shadow job only when the observer opts in by + /// returning `true`. The bundled metric exporters opt in. fn observes_shadow_outcomes(&self) -> bool { - true + false } } diff --git a/rust/src/runtime.rs b/rust/src/runtime.rs new file mode 100644 index 00000000..9514c7a4 --- /dev/null +++ b/rust/src/runtime.rs @@ -0,0 +1,40 @@ +//! Detached task admission and timers. + +use std::time::Duration; + +use futures::future::BoxFuture; + +/// Runs detached work the cache does not await and supplies its timers. +/// +/// Detached work is raw source or adapter work that outlives a caller's +/// deadline, and shadow validation jobs. Timers deliver deadlines; a +/// controlled runtime may deliver them on its own schedule. +pub trait Runtime: Send + Sync + 'static { + /// Start `task` concurrently with the caller. + fn spawn(&self, task: BoxFuture<'static, ()>); + /// Start `task` after work that is already runnable has progressed. + /// + /// Production runtimes treat this like [`Runtime::spawn`]. A controlled + /// test runtime runs deferred work only once everything else is blocked. + fn defer(&self, task: BoxFuture<'static, ()>) { + self.spawn(task) + } + /// A timer that completes once `duration` has passed. + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()>; +} + +/// Spawns onto the ambient tokio runtime and uses tokio timers. +#[cfg(feature = "tokio")] +#[derive(Debug, Clone, Copy, Default)] +pub struct TokioRuntime; + +#[cfg(feature = "tokio")] +impl Runtime for TokioRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + tokio::spawn(task); + } + + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { + Box::pin(tokio::time::sleep(duration)) + } +} diff --git a/rust/src/scope.rs b/rust/src/scope.rs index 9411cc53..66f8a92d 100644 --- a/rust/src/scope.rs +++ b/rust/src/scope.rs @@ -20,6 +20,12 @@ pub(crate) struct Owner { pub(crate) state: Mutex, } +impl fmt::Debug for Owner { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Owner").field("live", &self.is_live()).finish() + } +} + impl Owner { pub(crate) fn new() -> Arc { Arc::new(Owner { diff --git a/rust/src/shadow.rs b/rust/src/shadow.rs index d6d229e3..7decd59d 100644 --- a/rust/src/shadow.rs +++ b/rust/src/shadow.rs @@ -73,7 +73,7 @@ impl Execution { ) -> ValueResult { let start = self.elapsed(); let source: Settled = start_pending( - self.core.spawner.as_ref(), + self.core.runtime.as_ref(), { let x = self.clone(); async move { x.source(layer).await } @@ -125,7 +125,7 @@ impl Execution { self.error_event(Layer::Remote, ErrorKind::ConfigResolution, false); } let x = self.clone(); - self.core.spawner.defer(Box::pin(async move { + self.core.runtime.defer(Box::pin(async move { x.run_shadow(flight, frame, source, started).await })); } @@ -186,7 +186,7 @@ impl Execution { .unwrap_or(DEFAULT_FALLBACK_TIMEOUT_MS); let reads: Arc>>> = Arc::new(Mutex::new(Vec::new())); let validation: Settled = start_pending( - self.core.spawner.as_ref(), + self.core.runtime.as_ref(), { let x = self.clone(); let flight = flight.clone(); @@ -210,9 +210,9 @@ impl Execution { // write work already keeps this operation running until raw completion. let pending = std::mem::take(&mut *reads.lock()); let core = x.core.clone(); - let spawner = core.spawner.clone(); + let runtime = core.runtime.clone(); let key = x.keys.logical.clone(); - spawner.spawn(Box::pin(async move { + runtime.spawn(Box::pin(async move { for read in pending { let _ = read.wait().await; } @@ -232,6 +232,7 @@ impl Execution { ); let verdict = await_deadline( clock.as_ref(), + self.core.runtime.as_ref(), &validation, started, Some(budget_ms), @@ -345,7 +346,7 @@ impl Execution { if result.is_ok() { // Let the caller finish its own continuation before any shadow // decode, comparison or dump work. - yield_deferred(self.core.spawner.as_ref()).await; + yield_deferred(self.core.runtime.as_ref()).await; } result } diff --git a/rust/src/spawn.rs b/rust/src/spawn.rs deleted file mode 100644 index b1310ed1..00000000 --- a/rust/src/spawn.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! Detached task admission. - -use futures::future::BoxFuture; - -/// Runs detached work the cache does not await: raw source or adapter work -/// that outlives a caller's deadline, and shadow validation jobs. -pub trait Spawner: Send + Sync + 'static { - /// Start `task` concurrently with the caller. - fn spawn(&self, task: BoxFuture<'static, ()>); - /// Start `task` after work that is already runnable has progressed. - /// - /// Production runtimes treat this like [`Spawner::spawn`]. A controlled - /// test executor runs deferred work only once everything else is blocked. - fn defer(&self, task: BoxFuture<'static, ()>) { - self.spawn(task) - } -} - -/// Spawns onto the ambient tokio runtime. -#[cfg(feature = "tokio")] -#[derive(Debug, Clone, Copy, Default)] -pub struct TokioSpawner; - -#[cfg(feature = "tokio")] -impl Spawner for TokioSpawner { - fn spawn(&self, task: BoxFuture<'static, ()>) { - tokio::spawn(task); - } -} diff --git a/rust/src/testing.rs b/rust/src/testing.rs new file mode 100644 index 00000000..816f75e5 --- /dev/null +++ b/rust/src/testing.rs @@ -0,0 +1,330 @@ +//! Deterministic test doubles: a virtual clock and a single-threaded runtime +//! that runs the cache's detached work to quiescence on demand (feature `test-util`). +//! +//! The cache's detached work and timers go through [`Runtime`], so a +//! single-threaded pool can run every task to quiescence after each command +//! (the `causally-ready-v1` settlement of the formal replay) and deliver timers +//! only when a test advances the clock. + +use std::future::Future; +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::Arc; +use std::task::{Context, Poll, Waker}; +use std::time::Duration; + +use crate::clock::Clock; +use crate::runtime::Runtime; +use futures::executor::{LocalPool, LocalSpawner}; +use futures::future::BoxFuture; +use futures::task::LocalSpawnExt; +use parking_lot::Mutex; + +/// Epoch of every controlled history: 2026-09-08T12:00:00.000Z. +pub const WALL_EPOCH_MS: i64 = 1_788_868_800_000; + +impl std::fmt::Debug for VirtualClock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("VirtualClock").field("wall_ms", &self.wall_ms()).field("elapsed", &self.elapsed()).finish() + } +} + +struct TimerState { + at_ns: u128, + id: u64, + fired: AtomicBool, + cancelled: AtomicBool, + waker: Mutex>, +} + +struct ClockState { + wall_ns: i128, + elapsed_ns: u128, + scheduler_ns: u128, + next_timer: u64, + timers: Vec>, +} + +/// Virtual wall, elapsed and scheduler clocks. Scheduler time is what timers +/// are registered against; it moves only when a history delivers timers, so a +/// silent shift never fires a deliberately held timer. +pub struct VirtualClock { + state: Mutex, +} + +impl VirtualClock { + pub fn new(wall_ms: i64) -> Arc { + Arc::new(VirtualClock { + state: Mutex::new(ClockState { + wall_ns: wall_ms as i128 * 1_000_000, + elapsed_ns: 0, + scheduler_ns: 0, + next_timer: 0, + timers: Vec::new(), + }), + }) + } + + /// Move wall (and unless `wall_only`, elapsed) time without delivering timers. + pub fn shift(&self, delta_ms: i64, wall_only: bool) { + self.shift_ns(delta_ms as i128 * 1_000_000, wall_only); + } + + pub fn shift_ns(&self, delta_ns: i128, wall_only: bool) { + let mut state = self.state.lock(); + state.wall_ns += delta_ns; + if !wall_only { + state.elapsed_ns = (state.elapsed_ns as i128 + delta_ns).max(0) as u128; + } + } + + /// The earliest live timer due at or before `target_ns` scheduler time. + fn pop_due(&self, target_ns: u128) -> Option> { + let mut state = self.state.lock(); + state.timers.retain(|t| !t.cancelled.load(Ordering::SeqCst) && !t.fired.load(Ordering::SeqCst)); + let mut best: Option = None; + for (index, timer) in state.timers.iter().enumerate() { + if timer.at_ns <= target_ns { + let better = match best { + None => true, + Some(current) => { + let c = &state.timers[current]; + timer.at_ns < c.at_ns || (timer.at_ns == c.at_ns && timer.id < c.id) + } + }; + if better { + best = Some(index); + } + } + } + best.map(|index| state.timers.remove(index)) + } + + fn set_scheduler(&self, at_ns: u128) { + let mut state = self.state.lock(); + let delta = at_ns.saturating_sub(state.scheduler_ns); + state.wall_ns += delta as i128; + state.elapsed_ns += delta; + state.scheduler_ns = at_ns; + } + + fn scheduler_ns(&self) -> u128 { + self.state.lock().scheduler_ns + } + + pub fn pending_timers(&self) -> usize { + let state = self.state.lock(); + state.timers.iter().filter(|t| !t.cancelled.load(Ordering::SeqCst) && !t.fired.load(Ordering::SeqCst)).count() + } +} + +impl Clock for VirtualClock { + fn wall_ms(&self) -> i64 { + (self.state.lock().wall_ns.div_euclid(1_000_000)) as i64 + } + + fn elapsed(&self) -> Duration { + let ns = self.state.lock().elapsed_ns; + Duration::from_nanos(ns.min(u64::MAX as u128) as u64) + } +} + +impl VirtualClock { + /// Register a timer against scheduler time. + pub fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { + // A millisecond timer must never shorten a fractional remaining budget: + // round the delay up to whole milliseconds, as the reference ports do. + let delay_ns = duration.as_nanos(); + let delay_ms = delay_ns.div_ceil(1_000_000); + let mut state = self.state.lock(); + state.next_timer += 1; + let timer = Arc::new(TimerState { + at_ns: state.scheduler_ns + delay_ms * 1_000_000, + id: state.next_timer, + fired: AtomicBool::new(false), + cancelled: AtomicBool::new(false), + waker: Mutex::new(None), + }); + state.timers.push(timer.clone()); + Box::pin(Sleep { timer }) + } +} + +struct Sleep { + timer: Arc, +} + +impl Future for Sleep { + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + if self.timer.fired.load(Ordering::SeqCst) { + return Poll::Ready(()); + } + *self.timer.waker.lock() = Some(cx.waker().clone()); + Poll::Pending + } +} + +impl Drop for Sleep { + fn drop(&mut self) { + self.timer.cancelled.store(true, Ordering::SeqCst); + } +} + +/// Tasks handed to the spawner. Immediate tasks run on the next drain; +/// deferred tasks run only once nothing else is runnable. +#[derive(Default)] +struct Queues { + immediate: Vec>, + deferred: Vec>, +} + +/// The test runtime handle: collects tasks for the pool and registers timers +/// on the virtual clock. +pub struct StepRuntime { + queues: Mutex, + spawned: AtomicU64, + clock: Arc, +} + +impl StepRuntime { + pub fn new(clock: Arc) -> Arc { + Arc::new(StepRuntime { queues: Mutex::new(Queues::default()), spawned: AtomicU64::new(0), clock }) + } + + /// Tasks handed over so far, immediate and deferred. + pub fn spawned(&self) -> u64 { + self.spawned.load(Ordering::Relaxed) + } +} + +impl std::fmt::Debug for StepRuntime { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("StepRuntime").field("spawned", &self.spawned()).finish() + } +} + +impl Runtime for StepRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.spawned.fetch_add(1, Ordering::Relaxed); + self.queues.lock().immediate.push(task); + } + + fn defer(&self, task: BoxFuture<'static, ()>) { + self.spawned.fetch_add(1, Ordering::Relaxed); + self.queues.lock().deferred.push(task); + } + + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { + self.clock.sleep(duration) + } +} + +/// The controlled executor of one test: a single-threaded pool plus the +/// clock and runtime handle every cache instance under test shares. +pub struct TestExecutor { + pool: LocalPool, + local: LocalSpawner, + pub clock: Arc, + pub runtime: Arc, +} + +impl std::fmt::Debug for TestExecutor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("TestExecutor").field("wall_ms", &self.clock.wall_ms()).finish() + } +} + +impl TestExecutor { + pub fn new(wall_ms: i64) -> Self { + let pool = LocalPool::new(); + let local = pool.spawner(); + let clock = VirtualClock::new(wall_ms); + let runtime = StepRuntime::new(clock.clone()); + TestExecutor { pool, local, clock, runtime } + } + + /// Run a future to completion on the pool, draining detached work. + pub fn block_on(&mut self, future: impl Future) -> R { + let result = self.pool.run_until(future); + self.drain(); + result + } + + /// Spawn driver-owned work onto the pool. + pub fn spawn(&self, task: impl Future + 'static) { + self.local.spawn_local(task).expect("pool accepts tasks"); + } + + fn move_immediate(&mut self) -> bool { + let tasks = std::mem::take(&mut self.runtime.queues.lock().immediate); + let moved = !tasks.is_empty(); + for task in tasks { + self.local.spawn_local(task).expect("pool accepts tasks"); + } + moved + } + + fn move_deferred(&mut self) -> bool { + let tasks = std::mem::take(&mut self.runtime.queues.lock().deferred); + let moved = !tasks.is_empty(); + for task in tasks { + self.local.spawn_local(task).expect("pool accepts tasks"); + } + moved + } + + /// Run until every task is blocked on a gate, an undelivered timer or a + /// scope gate and nothing is runnable, including deferred work that + /// becomes runnable once everything else has stalled. + pub fn drain(&mut self) { + loop { + loop { + self.pool.run_until_stalled(); + if !self.move_immediate() { + break; + } + } + if !self.move_deferred() { + return; + } + } + } + + /// Advance elapsed and wall time by `ms`. With `deliver`, scheduler time + /// moves too and every timer due on the way fires in order, draining after + /// each; without it the clocks jump silently. + pub fn advance(&mut self, ms: i64, deliver: bool) { + if !deliver { + self.clock.shift(ms, false); + self.drain(); + return; + } + let target = self.clock.scheduler_ns() + (ms.max(0) as u128) * 1_000_000; + loop { + self.drain(); + match self.clock.pop_due(target) { + Some(timer) => { + self.clock.set_scheduler(timer.at_ns); + timer.fired.store(true, Ordering::SeqCst); + if let Some(waker) = timer.waker.lock().take() { + waker.wake(); + } + } + None => { + self.clock.set_scheduler(target); + break; + } + } + } + self.drain(); + } + + /// Advance by fractional microseconds without delivering timers (the + /// local-clock profile's environment ticks). + pub fn advance_micros(&mut self, micros: i64) { + self.clock.shift_ns(micros as i128 * 1_000, false); + self.drain(); + } +} From 5a677d1f1804a7a551637e4aa6d511cd98283c09 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 00:32:10 -0700 Subject: [PATCH 09/31] test(rust): share the vector fixtures module between the frame and key vector tests --- rust/src/clock.rs | 4 +- rust/src/engine.rs | 8 +- rust/src/execution.rs | 46 +++++--- rust/src/lib.rs | 14 ++- rust/src/scope.rs | 4 +- rust/src/testing.rs | 36 +++++-- rust/tests/formal/fixtures_tmp.rs | 169 ------------------------------ rust/tests/protocol_frames.rs | 2 +- 8 files changed, 83 insertions(+), 200 deletions(-) delete mode 100644 rust/tests/formal/fixtures_tmp.rs diff --git a/rust/src/clock.rs b/rust/src/clock.rs index 9711e482..6c91b47b 100644 --- a/rust/src/clock.rs +++ b/rust/src/clock.rs @@ -35,7 +35,9 @@ impl SystemClock { let process = *PROCESS_ORIGIN.get_or_init(Instant::now); let now = Instant::now(); let phase = now.saturating_duration_since(process).as_nanos() % 1_000_000; - let origin = now.checked_sub(Duration::from_nanos(phase as u64)).unwrap_or(now); + let origin = now + .checked_sub(Duration::from_nanos(phase as u64)) + .unwrap_or(now); SystemClock { origin } } } diff --git a/rust/src/engine.rs b/rust/src/engine.rs index e4572b1b..6a7b8069 100644 --- a/rust/src/engine.rs +++ b/rust/src/engine.rs @@ -25,9 +25,9 @@ use crate::operation::{downcast_value, erase_load, ErasedOperation, Operation, R use crate::policy::RuntimePolicy; use crate::protocol::CompressionConfig; use crate::remote::{InvalidateRequest, Remote}; +use crate::runtime::Runtime; use crate::scope::{Owner, Scope}; use crate::shadow::ShadowFlight; -use crate::runtime::Runtime; /// Resolves a sparse runtime policy overlay once per enabled call. pub type PolicyProvider = Arc< @@ -452,7 +452,11 @@ impl DialCache { /// ``` pub fn enable_guard(&self) -> ScopeGuard { let owner = Owner::new(); - let scope = Scope { cache_id: self.core.id, owner: Some(owner.clone()), enabled: true }; + let scope = Scope { + cache_id: self.core.id, + owner: Some(owner.clone()), + enabled: true, + }; ScopeGuard { scope, owner } } diff --git a/rust/src/execution.rs b/rust/src/execution.rs index ece59371..660f67fd 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -273,15 +273,22 @@ async fn source_with_budget( |message| Err(Error::Panic(message)), ); let use_case = labels.use_case.to_string(); - let result = await_deadline(clock.as_ref(), core.runtime.as_ref(), &pending, start, budget, || { - if let Some(flag) = timed_out { - flag.store(true, Ordering::SeqCst); - } - Err(Error::FallbackTimeout(Arc::new(FallbackTimeout { - use_case, - timeout_ms: budget.unwrap_or(0), - }))) - }) + let result = await_deadline( + clock.as_ref(), + core.runtime.as_ref(), + &pending, + start, + budget, + || { + if let Some(flag) = timed_out { + flag.store(true, Ordering::SeqCst); + } + Err(Error::FallbackTimeout(Arc::new(FallbackTimeout { + use_case, + timeout_ms: budget.unwrap_or(0), + }))) + }, + ) .await; if result.is_err() { core.emit(Event::Error { @@ -645,13 +652,20 @@ impl Execution { let use_case = self.labels.use_case.to_string(); let waited = raw.clone(); let bounded = Box::pin(async move { - let result = await_deadline(clock.as_ref(), runtime.as_ref(), &waited, start, Some(timeout_ms), || { - cancel.cancel(); - Err(Arc::new(RemoteReadTimeout { - use_case, - timeout_ms, - }) as SharedError) - }) + let result = await_deadline( + clock.as_ref(), + runtime.as_ref(), + &waited, + start, + Some(timeout_ms), + || { + cancel.cancel(); + Err(Arc::new(RemoteReadTimeout { + use_case, + timeout_ms, + }) as SharedError) + }, + ) .await; result.map(|read| normalize_read_result(read, tracked)) }); diff --git a/rust/src/lib.rs b/rust/src/lib.rs index f38b5ba0..154a36ad 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -36,14 +36,22 @@ mod use_case; pub use cancel::CancelToken; pub use clock::{Clock, SystemClock}; pub use codec::{Codec, FromSync, JsonCodec, Payload, SyncCodec}; -pub use engine::{CoalescingState, DialCache, DialCacheBuilder, PolicyProvider, ProcessCoalescingState, ScopeGuard}; +pub use engine::{ + CoalescingState, DialCache, DialCacheBuilder, PolicyProvider, ProcessCoalescingState, + ScopeGuard, +}; pub use error::{BoxError, ConfigError, Error, FallbackTimeout, RemoteReadTimeout, SharedError}; pub use identity::{normalize_args, ArgValue, Identity, IdentityError, Keys}; pub use local::{LocalEntry, LocalStore, LruLocalStore, StoredValue}; pub use observe::{Event, Labels, LogEvent, LogLevel, Logger, Observer, ShadowMismatchDetails}; pub use operation::{Comparator, Operation, Preview, RecoveryPredicate, SourceBudget}; -pub use policy::{Policy, PolicyDefaults, PolicyError, ResolvedPolicy, RuntimePolicy, ShadowPolicy}; -pub use remote::{Frame, InvalidateRequest, MissReason, ReadContext, ReadRequest, ReadResult, Remote, WriteRequest}; +pub use policy::{ + Policy, PolicyDefaults, PolicyError, ResolvedPolicy, RuntimePolicy, ShadowPolicy, +}; +pub use remote::{ + Frame, InvalidateRequest, MissReason, ReadContext, ReadRequest, ReadResult, Remote, + WriteRequest, +}; pub use runtime::Runtime; #[cfg(feature = "tokio")] pub use runtime::TokioRuntime; diff --git a/rust/src/scope.rs b/rust/src/scope.rs index 66f8a92d..86957c45 100644 --- a/rust/src/scope.rs +++ b/rust/src/scope.rs @@ -22,7 +22,9 @@ pub(crate) struct Owner { impl fmt::Debug for Owner { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Owner").field("live", &self.is_live()).finish() + f.debug_struct("Owner") + .field("live", &self.is_live()) + .finish() } } diff --git a/rust/src/testing.rs b/rust/src/testing.rs index 816f75e5..230df4c1 100644 --- a/rust/src/testing.rs +++ b/rust/src/testing.rs @@ -25,7 +25,10 @@ pub const WALL_EPOCH_MS: i64 = 1_788_868_800_000; impl std::fmt::Debug for VirtualClock { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("VirtualClock").field("wall_ms", &self.wall_ms()).field("elapsed", &self.elapsed()).finish() + f.debug_struct("VirtualClock") + .field("wall_ms", &self.wall_ms()) + .field("elapsed", &self.elapsed()) + .finish() } } @@ -81,7 +84,9 @@ impl VirtualClock { /// The earliest live timer due at or before `target_ns` scheduler time. fn pop_due(&self, target_ns: u128) -> Option> { let mut state = self.state.lock(); - state.timers.retain(|t| !t.cancelled.load(Ordering::SeqCst) && !t.fired.load(Ordering::SeqCst)); + state + .timers + .retain(|t| !t.cancelled.load(Ordering::SeqCst) && !t.fired.load(Ordering::SeqCst)); let mut best: Option = None; for (index, timer) in state.timers.iter().enumerate() { if timer.at_ns <= target_ns { @@ -114,7 +119,11 @@ impl VirtualClock { pub fn pending_timers(&self) -> usize { let state = self.state.lock(); - state.timers.iter().filter(|t| !t.cancelled.load(Ordering::SeqCst) && !t.fired.load(Ordering::SeqCst)).count() + state + .timers + .iter() + .filter(|t| !t.cancelled.load(Ordering::SeqCst) && !t.fired.load(Ordering::SeqCst)) + .count() } } @@ -190,7 +199,11 @@ pub struct StepRuntime { impl StepRuntime { pub fn new(clock: Arc) -> Arc { - Arc::new(StepRuntime { queues: Mutex::new(Queues::default()), spawned: AtomicU64::new(0), clock }) + Arc::new(StepRuntime { + queues: Mutex::new(Queues::default()), + spawned: AtomicU64::new(0), + clock, + }) } /// Tasks handed over so far, immediate and deferred. @@ -201,7 +214,9 @@ impl StepRuntime { impl std::fmt::Debug for StepRuntime { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("StepRuntime").field("spawned", &self.spawned()).finish() + f.debug_struct("StepRuntime") + .field("spawned", &self.spawned()) + .finish() } } @@ -232,7 +247,9 @@ pub struct TestExecutor { impl std::fmt::Debug for TestExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("TestExecutor").field("wall_ms", &self.clock.wall_ms()).finish() + f.debug_struct("TestExecutor") + .field("wall_ms", &self.clock.wall_ms()) + .finish() } } @@ -242,7 +259,12 @@ impl TestExecutor { let local = pool.spawner(); let clock = VirtualClock::new(wall_ms); let runtime = StepRuntime::new(clock.clone()); - TestExecutor { pool, local, clock, runtime } + TestExecutor { + pool, + local, + clock, + runtime, + } } /// Run a future to completion on the pool, draining detached work. diff --git a/rust/tests/formal/fixtures_tmp.rs b/rust/tests/formal/fixtures_tmp.rs deleted file mode 100644 index d99279a0..00000000 --- a/rust/tests/formal/fixtures_tmp.rs +++ /dev/null @@ -1,169 +0,0 @@ -//! TEMPORARY stand-in for `tests/formal/fixtures.rs`, which another agent -//! writes concurrently with the same function names. The integrator switches -//! `#[path = "formal/fixtures_tmp.rs"]` in `tests/protocol_frames.rs` to -//! `formal/fixtures.rs` and deletes this file. This copy performs no -//! provenance (source hash) checks. - -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; - -use serde_json::Value; - -/// Replaces every lone UTF-16 surrogate escape in a fixture file so the JSON -/// still parses; vector runners substitute the binding's input conversion. -pub const LONE_SURROGATE_MARKER: char = '\u{E000}'; - -/// A path relative to the repository root. -pub fn repo_path(relative: &str) -> PathBuf { - Path::new(env!("CARGO_MANIFEST_DIR")) - .join("..") - .join(relative) -} - -/// Parse a JSON file whose strings may contain lone UTF-16 surrogate escapes, -/// which `serde_json` rejects; each becomes [`LONE_SURROGATE_MARKER`]. -pub fn load_json_marking_lone_surrogates(path: impl AsRef) -> Value { - let path = path.as_ref(); - let raw = std::fs::read_to_string(path) - .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); - serde_json::from_str(&mark_lone_surrogates(&raw)) - .unwrap_or_else(|error| panic!("parse {}: {error}", path.display())) -} - -fn mark_lone_surrogates(raw: &str) -> String { - let bytes = raw.as_bytes(); - let mut out = String::with_capacity(raw.len()); - let mut index = 0; - let escape_at = |at: usize| -> Option { - let hex = bytes.get(at..at + 6)?; - if &hex[..2] != b"\\u" { - return None; - } - u32::from_str_radix(std::str::from_utf8(&hex[2..]).ok()?, 16).ok() - }; - while index < bytes.len() { - if bytes[index] != b'\\' { - let char_len = utf8_char_len(bytes[index]); - out.push_str(&raw[index..index + char_len]); - index += char_len; - continue; - } - match escape_at(index) { - Some(unit) if (0xD800..=0xDBFF).contains(&unit) => { - if escape_at(index + 6).is_some_and(|low| (0xDC00..=0xDFFF).contains(&low)) { - out.push_str(&raw[index..index + 12]); - index += 12; - } else { - out.push_str(&format!("\\u{:04X}", LONE_SURROGATE_MARKER as u32)); - index += 6; - } - } - Some(unit) if (0xDC00..=0xDFFF).contains(&unit) => { - out.push_str(&format!("\\u{:04X}", LONE_SURROGATE_MARKER as u32)); - index += 6; - } - Some(_) => { - out.push_str(&raw[index..index + 6]); - index += 6; - } - None => { - // Any other escape is two bytes: copy both so `\\` never hides a `\u`. - let end = (index + 2).min(bytes.len()); - out.push_str(&raw[index..end]); - index = end; - } - } - } - out -} - -fn utf8_char_len(lead: u8) -> usize { - match lead { - 0x00..=0x7F => 1, - 0xC0..=0xDF => 2, - 0xE0..=0xEF => 3, - _ => 4, - } -} - -/// The portable protocol vector groups: `formal/protocol-vectors.json` merged -/// with every Quint-generated protocol artifact listed in -/// `formal/execution.json`. `selection` is `"all"`, `"generated"` or -/// `"fixed"` (the values of `DIALCACHE_PROTOCOL_CORPUS`). -pub fn protocol_groups(selection: &str) -> BTreeMap> { - assert!( - matches!(selection, "all" | "generated" | "fixed"), - "unknown protocol corpus selection {selection}" - ); - let fixed = load_json_marking_lone_surrogates(repo_path("formal/protocol-vectors.json")); - assert_eq!( - fixed.get("schemaVersion").and_then(Value::as_u64), - Some(3), - "unsupported protocol vector schema" - ); - let mut groups: BTreeMap> = BTreeMap::new(); - for (name, rows) in fixed.as_object().expect("protocol vectors are an object") { - if name == "schemaVersion" { - continue; - } - let rows = rows - .as_array() - .unwrap_or_else(|| panic!("protocol group {name} is not an array")); - let entry = groups.entry(name.clone()).or_default(); - if selection != "generated" { - entry.extend(rows.iter().cloned()); - } - } - if selection == "fixed" { - return groups; - } - let manifest = load_json_marking_lone_surrogates(repo_path("formal/execution.json")); - let models = manifest - .get("models") - .and_then(Value::as_array) - .expect("execution.json lists models"); - for model in models { - let Some(export) = model.get("vectorExport") else { - continue; - }; - if export.get("kind").and_then(Value::as_str) != Some("protocol") { - continue; - } - let artifact = export - .get("artifact") - .and_then(Value::as_str) - .expect("protocol export names an artifact"); - let generated = load_json_marking_lone_surrogates(repo_path(artifact)); - assert_eq!( - generated.get("schemaVersion").and_then(Value::as_u64), - Some(3), - "{artifact}: unsupported schema" - ); - let mut count = 0; - for (name, rows) in generated - .as_object() - .expect("generated artifact is an object") - { - if name == "schemaVersion" || name == "provenance" { - continue; - } - let rows = rows - .as_array() - .unwrap_or_else(|| panic!("{artifact}: group {name} is not an array")); - let entry = groups - .get_mut(name) - .unwrap_or_else(|| panic!("{artifact}: unknown generated protocol group {name}")); - entry.extend(rows.iter().cloned()); - count += rows.len(); - } - let expected = export - .get("cases") - .and_then(Value::as_u64) - .expect("protocol export declares cases") as usize; - assert_eq!( - count, expected, - "{artifact}: incomplete generated protocol vector inventory" - ); - } - groups -} diff --git a/rust/tests/protocol_frames.rs b/rust/tests/protocol_frames.rs index c88777da..1e786de7 100644 --- a/rust/tests/protocol_frames.rs +++ b/rust/tests/protocol_frames.rs @@ -3,7 +3,7 @@ //! Quint-generated frame and envelope artifacts). // TODO(integrator): switch to `formal/fixtures.rs` and delete `formal/fixtures_tmp.rs`. -#[path = "formal/fixtures_tmp.rs"] +#[path = "formal/fixtures.rs"] mod fixtures; #[path = "formal/frame_vectors.rs"] mod frame_vectors; From 81e4fec2e6e7017c3bd99a6bd34a5b49b4548d47 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 00:58:44 -0700 Subject: [PATCH 10/31] test(rust): conformance harness with behavior, core and local-clock drivers, fixed scenarios and settlement control --- rust/src/clock.rs | 10 +- rust/src/error.rs | 9 +- rust/src/testing.rs | 61 +- rust/tests/conformance.rs | 392 +++++++- rust/tests/formal/core_driver.rs | 376 ++++++++ rust/tests/formal/driver.rs | 1529 ++++++++++++++++++++++++++++++ rust/tests/formal/gate.rs | 88 ++ rust/tests/formal/local_clock.rs | 122 +++ rust/tests/formal/mod.rs | 5 + rust/tests/formal/scenarios.rs | 88 ++ rust/tests/settlement_control.rs | 74 ++ 11 files changed, 2745 insertions(+), 9 deletions(-) create mode 100644 rust/tests/formal/core_driver.rs create mode 100644 rust/tests/formal/driver.rs create mode 100644 rust/tests/formal/gate.rs create mode 100644 rust/tests/formal/local_clock.rs create mode 100644 rust/tests/formal/scenarios.rs create mode 100644 rust/tests/settlement_control.rs diff --git a/rust/src/clock.rs b/rust/src/clock.rs index 6c91b47b..b99bf871 100644 --- a/rust/src/clock.rs +++ b/rust/src/clock.rs @@ -23,6 +23,13 @@ pub trait Clock: Send + Sync + 'static { static PROCESS_ORIGIN: OnceLock = OnceLock::new(); +/// The origin of a clock constructed `now_ns` after the process origin so that +/// its whole-millisecond readings fall on the process-wide grid: the nearest +/// aligned instant at or before construction. +pub fn grid_origin_ns(now_ns: u128) -> u128 { + now_ns - now_ns % 1_000_000 +} + /// The default clock: system wall time and a monotonic origin aligned to the /// process-wide millisecond grid, so default instances share one local expiry grid. #[derive(Debug, Clone)] @@ -34,7 +41,8 @@ impl SystemClock { pub fn new() -> Self { let process = *PROCESS_ORIGIN.get_or_init(Instant::now); let now = Instant::now(); - let phase = now.saturating_duration_since(process).as_nanos() % 1_000_000; + let since_process = now.saturating_duration_since(process).as_nanos(); + let phase = since_process - grid_origin_ns(since_process); let origin = now .checked_sub(Duration::from_nanos(phase as u64)) .unwrap_or(now); diff --git a/rust/src/error.rs b/rust/src/error.rs index 9399bc31..ed5cc60f 100644 --- a/rust/src/error.rs +++ b/rust/src/error.rs @@ -41,9 +41,14 @@ impl Error { } } - /// Whether this is the library's own source deadline error. + /// Whether this is a source deadline error: the library's own, or one a + /// source propagated from a nested cached call. pub fn is_fallback_timeout(&self) -> bool { - matches!(self, Error::FallbackTimeout(_)) + match self { + Error::FallbackTimeout(_) => true, + Error::Source(error) => error.downcast_ref::().is_some(), + _ => false, + } } } diff --git a/rust/src/testing.rs b/rust/src/testing.rs index 230df4c1..d6a2697f 100644 --- a/rust/src/testing.rs +++ b/rust/src/testing.rs @@ -267,11 +267,22 @@ impl TestExecutor { } } - /// Run a future to completion on the pool, draining detached work. - pub fn block_on(&mut self, future: impl Future) -> R { - let result = self.pool.run_until(future); + /// Run a future to completion on the pool, draining detached work as it + /// is spawned. Panics if the future stays blocked once nothing is runnable: + /// it would be waiting on a gate only the test can release. + pub fn block_on(&mut self, future: impl Future + 'static) -> R { + let slot: std::rc::Rc>> = + std::rc::Rc::new(std::cell::RefCell::new(None)); + let sink = slot.clone(); + self.local + .spawn_local(async move { + let result = future.await; + *sink.borrow_mut() = Some(result); + }) + .expect("pool accepts tasks"); self.drain(); - result + let result = slot.borrow_mut().take(); + result.expect("block_on future stayed blocked on a test-owned gate") } /// Spawn driver-owned work onto the pool. @@ -350,3 +361,45 @@ impl TestExecutor { self.drain(); } } + +/// A clock aligned to the shared millisecond grid of a [`VirtualClock`], the +/// way [`SystemClock`](crate::SystemClock) aligns to the process grid. +/// Instances constructed at different fractional times share one grid. +pub struct VirtualGridClock { + base: Arc, + origin_ns: u128, +} + +impl std::fmt::Debug for VirtualGridClock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("VirtualGridClock") + .field("origin_ns", &self.origin_ns) + .finish() + } +} + +impl VirtualGridClock { + pub fn new(base: Arc) -> Self { + let now_ns = base.state.lock().elapsed_ns; + VirtualGridClock { + base, + origin_ns: crate::clock::grid_origin_ns(now_ns), + } + } +} + +impl Clock for VirtualGridClock { + fn wall_ms(&self) -> i64 { + self.base.wall_ms() + } + + fn elapsed(&self) -> Duration { + let ns = self + .base + .state + .lock() + .elapsed_ns + .saturating_sub(self.origin_ns); + Duration::from_nanos(ns.min(u64::MAX as u128) as u64) + } +} diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index b2f5dc52..f51188b5 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -1,3 +1,391 @@ -fn main() { - println!("conformance harness placeholder"); +//! The Rust conformance harness: replays the shared corpus through the real +//! cache API against the Node replay coordinator, runs the fixed behavioral +//! scenarios and protocol vectors, and checks the shared witness evidence. +//! +//! Without selectors it replays the committed smoke histories. With the +//! `DIALCACHE_*_TRACE_DIR` selectors it replays the complete generated corpus +//! and writes the JSONL report named by `DIALCACHE_RUST_REPORT`. Node 24 must +//! be on `PATH` for the coordinator. + +#![allow(dead_code)] + +#[path = "formal/fixtures.rs"] +mod fixtures; +mod formal; +#[path = "formal/frame_vectors.rs"] +mod frame_vectors; +#[path = "formal/key_vectors.rs"] +mod key_vectors; + +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use formal::core_driver::CoreDriver; +use formal::driver::{install_panic_hook, Driver}; +use formal::inventory::{self, repo_root, Selection, TraceSource}; +use formal::local_clock::LocalClockDriver; +use formal::report::Report; +use formal::scenarios; +use formal::transport::{Coordinator, Prepared}; +use serde_json::{Map, Value}; + +const FEATURE_PROFILES: [&str; 13] = [ + "admission", + "independent", + "layers", + "local-failure", + "policy", + "recovery", + "recovery-read", + "runtime-boundaries", + "scope", + "shadow", + "shadow-layers", + "source-budgets", + "runtime-boundaries-placeholder", +]; + +struct Run { + selection: Selection, + report: Report, + coordinator: Coordinator, + profile_actions: BTreeMap>, + seen_actions: BTreeMap>, + failures: usize, + coverage_failures: Vec, +} + +fn main() -> ExitCode { + install_panic_hook(); + match run() { + Ok(true) => ExitCode::SUCCESS, + Ok(false) => ExitCode::FAILURE, + Err(error) => { + eprintln!("conformance harness failed: {error}"); + ExitCode::FAILURE + } + } +} + +fn run() -> Result { + let selection = Selection::from_env()?; + let mut report = Report::from_env()?; + report.start()?; + inventory::registry_check()?; + let mut coordinator = Coordinator::spawn()?; + let mut request = Map::new(); + request.insert("op".to_string(), Value::from("profiles")); + let info = coordinator.call(request)?; + let mut profile_actions = BTreeMap::new(); + if let Some(profiles) = info.get("profiles").and_then(Value::as_object) { + for (name, actions) in profiles { + let actions: BTreeSet = actions + .as_array() + .into_iter() + .flatten() + .filter_map(Value::as_str) + .map(str::to_string) + .collect(); + profile_actions.insert(name.clone(), actions); + } + } + let mut run = Run { + selection, + report, + coordinator, + profile_actions, + seen_actions: BTreeMap::new(), + failures: 0, + coverage_failures: Vec::new(), + }; + run.core()?; + run.effects()?; + run.features()?; + run.local_clock()?; + run.scenarios()?; + run.protocol_vectors()?; + run.witnesses()?; + for (profile, expected) in &run.profile_actions { + let selected_full = match profile.as_str() { + "core" => run.selection.core.is_directory(), + "effects" => run.selection.effects.is_directory(), + _ => { + run.selection.features.is_directory() + && run + .selection + .feature_profile + .as_deref() + .is_none_or(|p| p == profile) + } + }; + if !selected_full { + continue; + } + let seen = run.seen_actions.get(profile).cloned().unwrap_or_default(); + for action in expected { + if action != "init" && !seen.contains(action) { + run.coverage_failures + .push(format!("{profile} corpus omits action {action}")); + } + } + } + let summary = run.report.finish()?; + let Run { + coordinator, + coverage_failures, + .. + } = run; + coordinator.finish()?; + for failure in &coverage_failures { + eprintln!("coverage: {failure}"); + } + Ok(summary.failed == 0 && coverage_failures.is_empty()) +} + +fn note_actions(seen: &mut BTreeMap>, profile: &str, prepared: &Prepared) { + let entry = seen.entry(profile.to_string()).or_default(); + for action in &prepared.actions { + entry.insert(action.clone()); + } +} + +impl Run { + fn replay_core(&mut self, path: &Path) -> Result<(), String> { + let raw = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + let prepared = self.coordinator.prepare("core", path, Some(&raw))?; + note_actions(&mut self.seen_actions, "core", &prepared); + let mut driver = CoreDriver::new(); + let cell = std::cell::RefCell::new(&mut driver); + let mut apply = |input: &Value| cell.borrow_mut().apply(input); + let mut observation = || cell.borrow().observation(); + let mut wall = || cell.borrow().wall_ms(); + self.coordinator + .execute(&prepared, &mut apply, &mut observation, &mut wall, &mut []) + } + + fn core(&mut self) -> Result<(), String> { + let full = self.selection.core.is_directory(); + for path in self.selection.core_paths()? { + let id = inventory::trace_case_id("core", &path, full); + let outcome = self.replay_core(&path); + self.record(&id, outcome)?; + } + Ok(()) + } + + fn replay_behavior( + &mut self, + profile: &str, + path: &Path, + effects_monitor: bool, + ) -> Result<(), String> { + let prepared = self.coordinator.prepare(profile, path, None)?; + note_actions(&mut self.seen_actions, profile, &prepared); + let mut driver = Driver::new(prepared.fixture.clone()); + let result = { + let driver_ref = &mut driver; + let cell = std::cell::RefCell::new(driver_ref); + let mut apply = |input: &Value| cell.borrow_mut().apply(input); + let mut observation = || cell.borrow().observation(); + let mut wall = || cell.borrow().wall_ms(); + let mut monitor = || cell.borrow().assert_effects_history(); + let mut monitors: Vec<&mut dyn FnMut() -> Result<(), String>> = Vec::new(); + if effects_monitor { + monitors.push(&mut monitor); + } + self.coordinator.execute( + &prepared, + &mut apply, + &mut observation, + &mut wall, + &mut monitors, + ) + }; + driver.close(); + result + } + + fn effects(&mut self) -> Result<(), String> { + inventory::require_behavior_profile("effects")?; + let full = self.selection.effects.is_directory(); + for path in self.selection.effects_paths()? { + let id = inventory::trace_case_id("effects", &path, full); + let outcome = self.replay_behavior("effects", &path, true); + self.record(&id, outcome)?; + } + Ok(()) + } + + fn features(&mut self) -> Result<(), String> { + let mut names: Vec = self + .profile_actions + .keys() + .filter(|n| !matches!(n.as_str(), "core" | "effects" | "local-clock")) + .cloned() + .collect(); + names.sort(); + let full = self.selection.features.is_directory(); + for name in names { + if let Some(selected) = &self.selection.feature_profile { + if selected != &name { + continue; + } + } + let paths = self.selection.feature_paths(&name)?; + if !paths.is_empty() { + inventory::require_behavior_profile(&name)?; + } + for path in paths { + let id = inventory::trace_case_id(&name, &path, full); + let outcome = self.replay_behavior(&name, &path, false); + self.record(&id, outcome)?; + } + } + Ok(()) + } + + fn local_clock(&mut self) -> Result<(), String> { + let paths = self.selection.feature_paths("local-clock")?; + if paths.is_empty() { + return Ok(()); + } + if let Some(selected) = &self.selection.feature_profile { + if selected != "local-clock" { + return Ok(()); + } + } + inventory::require_behavior_profile("local-clock")?; + let full = self.selection.features.is_directory(); + for path in paths { + let id = inventory::trace_case_id("local-clock", &path, full); + let outcome = (|| { + let prepared = self.coordinator.prepare("local-clock", &path, None)?; + note_actions(&mut self.seen_actions, "local-clock", &prepared); + let mut driver = LocalClockDriver::new(); + let cell = std::cell::RefCell::new(&mut driver); + let mut apply = |input: &Value| cell.borrow_mut().apply(input); + let mut observation = || cell.borrow().observation(); + let mut wall = || formal::report::now_ms(); + self.coordinator.execute( + &prepared, + &mut apply, + &mut observation, + &mut wall, + &mut [], + ) + })(); + self.record(&id, outcome)?; + } + Ok(()) + } + + fn scenarios(&mut self) -> Result<(), String> { + let scenarios = scenarios::load_scenarios()?; + let filter = self.selection.behavior_scenario.clone(); + let mut matched = 0; + for scenario in &scenarios { + let name = scenario.get("name").and_then(Value::as_str).unwrap_or(""); + let feature = scenario + .get("feature") + .and_then(Value::as_str) + .unwrap_or(""); + if let Some(filter) = &filter { + if !name.contains(filter.as_str()) { + continue; + } + } + matched += 1; + let id = inventory::scenario_case_id(feature, name); + let outcome = scenarios::replay_scenario(scenario); + self.record(&id, outcome)?; + } + if matched == 0 { + return Err("no behavioral scenario matched".to_string()); + } + Ok(()) + } + + fn protocol_vectors(&mut self) -> Result<(), String> { + let selection = match self.selection.protocol_corpus { + inventory::ProtocolCorpus::All => "all", + inventory::ProtocolCorpus::Generated => "generated", + inventory::ProtocolCorpus::Fixed => "fixed", + }; + let groups = fixtures::protocol_groups(selection); + for (group, vectors) in &groups { + let check: fn(&Value) -> Result<(), String> = match group.as_str() { + "keyVectors" => key_vectors::check_key_vector, + "invalidKeyVectors" => key_vectors::check_invalid_key_vector, + "normalizeArgsVectors" => key_vectors::check_normalize_args_vector, + "rampVectors" => key_vectors::check_ramp_vector, + "frameVectors" => frame_vectors::check_frame_vector, + "trackedDecodeVectors" => frame_vectors::check_tracked_decode_vector, + "untrackedDecodeVectors" => frame_vectors::check_untracked_decode_vector, + "invalidTimestampVectors" => frame_vectors::check_invalid_timestamp_vector, + "durationVectors" => frame_vectors::check_duration_vector, + "envelopeVectors" => frame_vectors::check_envelope_vector, + "compressedDecodeVectors" => frame_vectors::check_compressed_decode_vector, + "compressionWriteVectors" => frame_vectors::check_compression_write_vector, + other => return Err(format!("unknown protocol vector group {other}")), + }; + for vector in vectors { + let name = vector.get("name").and_then(Value::as_str).unwrap_or(""); + let id = inventory::protocol_case_id(group, name); + let outcome = check(vector); + self.record(&id, outcome)?; + } + } + Ok(()) + } + + fn witnesses(&mut self) -> Result<(), String> { + let mut profiles: BTreeMap> = BTreeMap::new(); + if self.selection.effects.is_directory() { + profiles.insert("effects".to_string(), self.selection.effects_paths()?); + } + if self.selection.features.is_directory() && self.selection.feature_profile.is_none() { + for name in self.profile_actions.keys() { + if matches!(name.as_str(), "core" | "effects") { + continue; + } + profiles.insert(name.clone(), self.selection.feature_paths(name)?); + } + } + if profiles.is_empty() { + return Ok(()); + } + let Some(directory) = self.selection.witness_evidence_dir.clone() else { + return Err("full generated replay requires DIALCACHE_WITNESS_EVIDENCE_DIR with matching evaluated witness evidence".to_string()); + }; + let root = repo_root(); + for (profile, paths) in profiles { + let id = inventory::witness_case_id(&profile); + let outcome = + formal::witness::check_witness_evidence(&root, &profile, &directory, &paths); + self.record(&id, outcome)?; + } + Ok(()) + } + + fn record(&mut self, id: &str, outcome: Result<(), String>) -> Result<(), String> { + let started = formal::report::now_ms(); + if let Err(message) = &outcome { + self.failures += 1; + eprintln!("FAIL {id}\n{message}"); + } + self.report + .case(id, &outcome, started, formal::report::now_ms())?; + Ok(()) + } +} + +#[allow(dead_code)] +fn _profiles() -> &'static [&'static str] { + &FEATURE_PROFILES +} + +#[allow(dead_code)] +fn _trace(source: &TraceSource) -> bool { + source.is_directory() } diff --git a/rust/tests/formal/core_driver.rs b/rust/tests/formal/core_driver.rs new file mode 100644 index 00000000..8c9fb0b7 --- /dev/null +++ b/rust/tests/formal/core_driver.rs @@ -0,0 +1,376 @@ +//! The core profile driver: integer values, an in-memory remote and a wall +//! clock that moves only through `advanceWall`. + +use std::collections::HashMap; +use std::sync::Arc; + +use dialcache::protocol::{decode_frame, encode_frame}; +use dialcache::testing::{TestExecutor, VirtualClock}; +use dialcache::{ + BoxError, Clock, DialCache, Error, Identity, InvalidateRequest, Operation, Policy, ReadContext, + ReadRequest, ReadResult, Remote, Scope, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; +use serde_json::{json, Map, Value}; + +use super::driver::WALL_EPOCH_MS; +use super::gate::Gate; + +struct Entry { + raw: Vec, + expires: i64, +} + +#[derive(Default)] +struct RemoteState { + values: HashMap, + watermarks: HashMap, + reads: u64, + writes: u64, + read_failure: bool, + pub discard_writes: bool, +} + +struct MemoryRemote { + state: Arc>, + clock: Arc, +} + +impl Remote for MemoryRemote { + fn read( + &self, + request: ReadRequest, + _context: ReadContext, + ) -> BoxFuture<'_, Result> { + let state = self.state.clone(); + let clock = self.clock.clone(); + Box::pin(async move { + let mut s = state.lock(); + s.reads += 1; + if s.read_failure { + return Err("controlled read failure".into()); + } + let elapsed = clock.elapsed_ms(); + let raw = s + .values + .get(&request.value_key) + .filter(|entry| elapsed < entry.expires) + .map(|entry| entry.raw.clone()); + let watermark = request + .watermark_key + .as_ref() + .and_then(|key| s.watermarks.get(key).cloned()); + let tracked = request.watermark_key.is_some(); + Ok(decode_frame( + raw.as_deref(), + tracked, + watermark.as_deref().map(str::as_bytes), + )?) + }) + } + + fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + let state = self.state.clone(); + let clock = self.clock.clone(); + Box::pin(async move { + let mut s = state.lock(); + s.writes += 1; + if s.discard_writes { + return Ok(()); + } + let raw = encode_frame(&request.frame)?; + let expires = clock.elapsed_ms() + request.ttl_ms as i64; + s.values.insert(request.value_key, Entry { raw, expires }); + Ok(()) + }) + } + + fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + let state = self.state.clone(); + Box::pin(async move { + let mut s = state.lock(); + s.writes += 1; + let mut cutoff = request.invalidated_at_ms + request.future_buffer_ms; + let old: u64 = s + .watermarks + .get(&request.watermark_key) + .and_then(|w| w.parse().ok()) + .unwrap_or(0); + if cutoff < old { + cutoff = old; + } + s.watermarks + .insert(request.watermark_key, cutoff.to_string()); + Ok(()) + }) + } +} + +/// The core driver over one cache instance. +pub struct CoreDriver { + pub exec: TestExecutor, + cache: DialCache, + remote: Arc>, + clock: Arc, + source: Arc>, + last: i64, + loaders: Arc>>, +} + +impl std::fmt::Debug for CoreDriver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CoreDriver").finish() + } +} + +fn text(value: Option<&Value>) -> String { + value.and_then(Value::as_str).unwrap_or("").to_string() +} + +impl CoreDriver { + pub fn new() -> CoreDriver { + Self::with_discarded_writes(false) + } + + /// A broken driver whose remote acknowledges writes it never stores. + pub fn with_discarded_writes(discard: bool) -> CoreDriver { + let exec = TestExecutor::new(WALL_EPOCH_MS); + let clock = exec.clock.clone(); + let remote = Arc::new(Mutex::new(RemoteState { + discard_writes: discard, + ..RemoteState::default() + })); + let cache = DialCache::builder() + .clock_arc(clock.clone()) + .runtime_arc(exec.runtime.clone()) + .remote_arc(Arc::new(MemoryRemote { + state: remote.clone(), + clock: clock.clone(), + })) + .local_capacity(10_000) + .build() + .expect("core driver configuration"); + CoreDriver { + exec, + cache, + remote, + clock, + source: Arc::new(Mutex::new(1)), + last: 0, + loaders: Arc::new(Mutex::new(HashMap::new())), + } + } + + pub fn wall_ms(&self) -> i64 { + self.clock.wall_ms() + } + + fn operation(identity: Identity, policy: &Value) -> Result, String> { + let policy = Policy::from_json(policy).map_err(|e| e.to_string())?; + Ok(Operation::::new(identity).policy(policy)) + } + + fn loader( + &self, + counter: &str, + gate: Option>, + started: Option>>, + ) -> impl Fn(Scope) -> BoxFuture<'static, Result> + Send + Sync + Clone + 'static + { + let loaders = self.loaders.clone(); + let source = self.source.clone(); + let counter = counter.to_string(); + move |_scope: Scope| { + let loaders = loaders.clone(); + let source = source.clone(); + let counter = counter.clone(); + let gate = gate.clone(); + let started = started.clone(); + Box::pin(async move { + let value = { + *loaders.lock().entry(counter).or_insert(0) += 1; + *source.lock() + }; + if let Some(started) = started { + *started.lock() += 1; + } + if let Some(gate) = gate { + gate.wait().await; + } + Ok(value) + }) + } + } + + /// Apply one core command. Only the selected public action reaches the + /// driver; publication is observed through later actual calls. + pub fn apply(&mut self, input: &Value) -> Result<(), String> { + match text(input.get("op")).as_str() { + "advanceWall" => { + let ms = input.get("ms").and_then(Value::as_i64).unwrap_or(0); + self.clock.shift(ms, true); + Ok(()) + } + "bumpSource" => { + *self.source.lock() += 1; + Ok(()) + } + "invalidate" => { + let identity = input.get("identity").cloned().unwrap_or(Value::Null); + let cache = self.cache.clone(); + let key_type = text(identity.get("keyType")); + let id = text(identity.get("id")); + self.exec + .block_on(async move { cache.invalidate(&key_type, &id, 0).await }) + .map_err(|e| e.to_string()) + } + "call" => self.call(input), + other => Err(format!("unknown core driver command {other}")), + } + } + + fn call(&mut self, input: &Value) -> Result<(), String> { + let identity_json = input.get("identity").cloned().unwrap_or(Value::Null); + let identity = Identity::new( + text(identity_json.get("keyType")), + text(identity_json.get("id")), + text(identity_json.get("useCase")), + ) + .namespace("urn") + .tracked(identity_json.get("tracked") == Some(&Value::Bool(true))); + let operation = Self::operation(identity, input.get("policy").unwrap_or(&Value::Null))?; + let counter = text(input.get("counter")); + let read_failure = input.get("readFailure") == Some(&Value::Bool(true)); + self.remote.lock().read_failure = read_failure; + let outcome: Result = match text(input.get("mode")).as_str() { + "outside" => { + let cache = self.cache.clone(); + let load = self.loader(&counter, None, None); + self.exec.block_on(async move { + cache + .get_or_load(&Scope::outside(), operation, load) + .await + .map(|v| *v) + }) + } + "single" => { + let cache = self.cache.clone(); + let load = self.loader(&counter, None, None); + self.exec.block_on(async move { + let inner = cache.clone(); + cache + .enable(|scope| async move { + inner.get_or_load(&scope, operation, load).await.map(|v| *v) + }) + .await + }) + } + "request-pair" => { + let cache = self.cache.clone(); + let load = self.loader(&counter, None, None); + self.exec.block_on(async move { + let inner = cache.clone(); + cache + .enable(|scope| async move { + let first = inner + .get_or_load(&scope, operation.clone(), load.clone()) + .await?; + let second = inner.get_or_load(&scope, operation, load).await?; + if *first != *second { + return Err(Error::Config(dialcache::ConfigError::Invalid( + "request pair differs".to_string(), + ))); + } + Ok(*first) + }) + .await + }) + } + "coalesced-pair" => self.pair(operation, &counter), + other => Err(Error::Config(dialcache::ConfigError::Invalid(format!( + "unknown core call mode {other}" + )))), + }; + self.remote.lock().read_failure = false; + match outcome { + Ok(value) => { + self.last = value; + Ok(()) + } + Err(error) => Err(error.to_string()), + } + } + + /// Two enabled callers whose leader source stays open until both have + /// started, so overlap is real. A warm cache completes the first call + /// before the second starts; lost sharing stays observable in loader counts. + fn pair(&mut self, operation: Operation, counter: &str) -> Result { + let gate: Gate<()> = Gate::new(); + let started = Arc::new(Mutex::new(0u64)); + let results: Arc>>> = Arc::new(Mutex::new(Vec::new())); + let results_handle = results.clone(); + let load = self.loader(counter, Some(gate.clone()), Some(started.clone())); + let cache = self.cache.clone(); + let make_call = move || { + let cache = cache.clone(); + let operation = operation.clone(); + let load = load.clone(); + let results = results.clone(); + async move { + let inner = cache.clone(); + let result = cache + .enable(|scope| async move { + inner.get_or_load(&scope, operation, load).await.map(|v| *v) + }) + .await; + results.lock().push(result); + } + }; + self.exec.spawn(make_call()); + self.exec.drain(); + self.exec.spawn(make_call()); + self.exec.drain(); + gate.settle(()); + self.exec.drain(); + let results = std::mem::take(&mut *results_handle.lock()); + if results.len() != 2 { + return Err(Error::Config(dialcache::ConfigError::Invalid( + "coalesced pair did not complete".to_string(), + ))); + } + let mut values = Vec::new(); + for result in results { + values.push(result?); + } + if values[0] != values[1] { + return Err(Error::Config(dialcache::ConfigError::Invalid( + "pair returned different values".to_string(), + ))); + } + Ok(values[0]) + } + + pub fn observation(&self) -> Value { + let remote = self.remote.lock(); + let loaders = self.loaders.lock(); + let mut out = Map::new(); + out.insert("sourceVersion".to_string(), json!(*self.source.lock())); + out.insert("lastResult".to_string(), json!(self.last)); + out.insert("redisReads".to_string(), json!(remote.reads)); + out.insert("redisWrites".to_string(), json!(remote.writes)); + for field in [ + "outsideLoaderCalls", + "requestLoaderCalls", + "localLoaderCalls", + "coalescedLoaderCalls", + "remoteLoaderCalls", + ] { + out.insert( + field.to_string(), + json!(loaders.get(field).copied().unwrap_or(0)), + ); + } + Value::Object(out) + } +} diff --git a/rust/tests/formal/driver.rs b/rust/tests/formal/driver.rs new file mode 100644 index 00000000..1e83ba4d --- /dev/null +++ b/rust/tests/formal/driver.rs @@ -0,0 +1,1529 @@ +//! The behavior driver: executes the shared replay commands against the real +//! Rust cache under the deterministic test executor. It controls external +//! effects only (sources, codecs, the fake remote, policy replies, clocks, +//! faults) and reports actual observations; no expected state enters it. + +use std::collections::HashMap; +use std::fmt; +use std::sync::Arc; + +use dialcache::observe::{Event, LogEvent, Logger, Observer}; +use dialcache::protocol::{decode_frame, encode_frame, read_result_from_untrusted_json}; +use dialcache::testing::{TestExecutor, VirtualClock}; +use dialcache::{ + BoxError, Codec, DialCache, Error, FallbackTimeout, Identity, InvalidateRequest, LocalEntry, + LocalStore, LruLocalStore, Operation, Payload, Policy, ReadContext, ReadRequest, ReadResult, + Remote, RuntimePolicy, Scope, SourceBudget, StoredValue, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; +use serde_json::{json, Map, Value}; + +use super::gate::Gate; +use super::json::json_equal; + +/// Epoch of every controlled history: 2026-09-08T12:00:00.000Z. +pub const WALL_EPOCH_MS: i64 = 1_788_868_800_000; + +/// Panic payload of every controlled callback failure; the harness panic hook +/// keeps it out of the test output. +pub struct ControlledFailure; + +pub fn controlled_panic() -> ! { + std::panic::panic_any(ControlledFailure) +} + +/// Install a panic hook that stays silent for controlled failures. +pub fn install_panic_hook() { + let default = std::panic::take_hook(); + std::panic::set_hook(Box::new(move |info| { + if info.payload().downcast_ref::().is_none() { + default(info); + } + })); +} + +/// The N-th controlled source failure. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("source failure {0}")] +pub struct SourceFailure(pub usize); + +/// The controlled mutation failure returned by the fake remote. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("controlled mutation failure")] +pub struct MaintenanceFailure; + +/// The cached value domain of the histories: JSON primitives plus absence. +#[derive(Debug, Clone)] +pub enum Val { + Absent, + Json(Value), +} + +impl Val { + fn from_json(value: Value) -> Val { + Val::Json(value) + } + + /// The observation encoding: `{absent:true}` for absence, else the primitive. + pub fn observed(&self) -> Value { + match self { + Val::Absent => json!({ "absent": true }), + Val::Json(value) => value.clone(), + } + } + + pub fn json_preview(&self) -> Option { + match self { + Val::Absent => None, + Val::Json(value) => serde_json::to_string(value).ok(), + } + } +} + +impl PartialEq for Val { + fn eq(&self, other: &Val) -> bool { + match (self, other) { + (Val::Absent, Val::Absent) => true, + (Val::Json(a), Val::Json(b)) => json_equal(a, b), + _ => false, + } + } +} + +#[derive(Clone)] +enum LoaderOutcome { + Value(Val), + Fail(usize), + Timeout(usize), +} + +struct Stored { + raw: Vec, + expires: i64, +} + +#[derive(Clone, Debug)] +pub struct HistoryEvent { + pub event: &'static str, + pub id: usize, + pub at: i64, + pub outcome: &'static str, + pub duration_ms: f64, + pub failed: bool, +} + +pub struct Shared { + fixture: Value, + observed: Map, + loaders: Vec>, + timeout_errors: Vec, + faults: HashMap, + runtime_policy: Value, + values: HashMap, + reply: Option, + effects: HashMap>>>, + history: Vec, + fallback_failed: bool, + pub discard_writes: bool, + pub discard_invalidations: bool, + unsettled: Option>, +} + +impl Shared { + fn increment(&mut self, field: &str) -> usize { + let current = self + .observed + .get(field) + .and_then(Value::as_u64) + .unwrap_or(0); + self.observed + .insert(field.to_string(), Value::from(current + 1)); + current as usize + } + + fn push(&mut self, field: &str, value: Value) { + if let Some(Value::Array(items)) = self.observed.get_mut(field) { + items.push(value); + } + } + + fn fault(&self, name: &str) -> bool { + self.faults.get(name).copied().unwrap_or(false) + } + + fn observes(&self, kind: &str) -> bool { + self.fixture + .get("observe") + .and_then(Value::as_array) + .is_some_and(|kinds| kinds.iter().any(|k| k.as_str() == Some(kind))) + } + + fn record(&mut self, kind: &str, fields: Map) { + if !self.observes(kind) { + return; + } + let mut event = Map::new(); + event.insert("event".to_string(), Value::from(kind)); + for (k, v) in fields { + event.insert(k, v); + } + self.push("events", Value::Object(event)); + } + + fn observer_fails(&self) -> bool { + self.fixture.get("observerFailure") == Some(&Value::Bool(true)) || self.fault("observer") + } + + fn raw(&mut self, key: &str, elapsed_ms: i64) -> Option> { + let expired = match self.values.get(key) { + None => return None, + Some(item) => item.expires <= elapsed_ms, + }; + if expired { + self.values.remove(key); + return None; + } + self.values.get(key).map(|item| item.raw.clone()) + } + + fn timeout_index(&mut self, error: &Error) -> usize { + let ptr = |error: &Error| -> usize { + match error { + Error::FallbackTimeout(arc) => Arc::as_ptr(arc) as *const () as usize, + Error::Source(arc) => Arc::as_ptr(arc) as *const () as usize, + _ => 0, + } + }; + let wanted = ptr(error); + if let Some(index) = self.timeout_errors.iter().position(|e| ptr(e) == wanted) { + return index; + } + self.timeout_errors.push(error.clone()); + self.timeout_errors.len() - 1 + } + + fn classify_error(&mut self, error: &Error) -> String { + match error { + Error::Source(source) => { + if let Some(SourceFailure(id)) = source.downcast_ref::() { + format!("source:{id}") + } else if let Some(timeout) = source.downcast_ref::() { + // The source returned a deadline error of its own (a nested call's); + // it is that loader's failure, not this call's deadline. + match timeout.use_case.strip_prefix("NestedSource:") { + Some(id) => format!("source:{id}"), + None => format!("timeout:{}", self.timeout_index(error)), + } + } else { + format!("unexpected:{error}") + } + } + Error::FallbackTimeout(_) => format!("timeout:{}", self.timeout_index(error)), + other => format!("unexpected:{other}"), + } + } +} + +/// The empty observation record for a fixture: `events` only when it observes. +pub fn empty_observation(fixture: &Value) -> Map { + let mut o = Map::new(); + if fixture.get("observe").is_some() { + o.insert("events".to_string(), json!([])); + } + o.insert("calls".to_string(), json!([])); + for key in [ + "loaders", + "reads", + "writes", + "invalidations", + "loads", + "dumps", + "policyCalls", + "classifications", + "comparisons", + ] { + o.insert(key.to_string(), json!(0)); + } + for key in [ + "maintenance", + "sourceScopes", + "writeTtls", + "shadow", + "recovery", + ] { + o.insert(key.to_string(), json!([])); + } + o +} + +struct ScopeHandle { + instance: String, + scope: Arc>>, + gate: Gate<()>, + done: Gate<()>, +} + +/// One history's driver. +pub struct Driver { + pub exec: TestExecutor, + shared: Arc>, + clock: Arc, + fixture: Value, + instances: HashMap, + scopes: HashMap, + /// Harness control only: report the observation held before the + /// settlement drain. Conformance replays never set it. + pub skip_settle: bool, +} + +impl fmt::Debug for Driver { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Driver") + .field("instances", &self.instances.len()) + .finish() + } +} + +fn number(value: Option<&Value>) -> i64 { + match value { + Some(Value::Number(n)) => n + .as_i64() + .or_else(|| n.as_f64().map(|f| f as i64)) + .unwrap_or(0), + _ => 0, + } +} + +fn text(value: Option<&Value>) -> String { + value.and_then(Value::as_str).unwrap_or("").to_string() +} + +fn shared_labels(shared: &Shared) -> Map { + let _ = shared; + Map::new() +} + +impl Driver { + pub fn new(fixture: Value) -> Driver { + let exec = TestExecutor::new(WALL_EPOCH_MS); + let clock = exec.clock.clone(); + let shared = Shared { + observed: empty_observation(&fixture), + fixture: fixture.clone(), + loaders: Vec::new(), + timeout_errors: Vec::new(), + faults: HashMap::new(), + runtime_policy: json!({}), + values: HashMap::new(), + reply: None, + effects: HashMap::new(), + history: Vec::new(), + fallback_failed: false, + discard_writes: false, + discard_invalidations: false, + unsettled: None, + }; + let mut driver = Driver { + exec, + shared: Arc::new(Mutex::new(shared)), + clock, + fixture, + instances: HashMap::new(), + scopes: HashMap::new(), + skip_settle: false, + }; + driver.instance("default"); + driver + } + + pub fn shared(&self) -> &Arc> { + &self.shared + } + + pub fn wall_ms(&self) -> i64 { + use dialcache::Clock; + self.clock.wall_ms() + } + + fn elapsed_ms(&self) -> i64 { + use dialcache::Clock; + self.clock.elapsed_ms() + } + + fn identity(&self, key: &str, use_case: &str) -> Identity { + let key = if key.is_empty() { "1" } else { key }; + let use_case = if use_case.is_empty() { + "Behavior" + } else { + use_case + }; + Identity::new("id", key, use_case) + .namespace("urn") + .tracked(self.fixture.get("tracked") == Some(&Value::Bool(true))) + } + + fn classifier(&self, mode: &str) -> impl Fn(&Error) -> bool + Send + Sync + 'static { + let shared = self.shared.clone(); + let mode = mode.to_string(); + move |_error: &Error| { + shared.lock().increment("classifications"); + if mode == "error" { + controlled_panic(); + } + mode == "allow" + } + } + + fn instance(&mut self, id: &str) -> DialCache { + if let Some(cache) = self.instances.get(id) { + return cache.clone(); + } + let fixture = &self.fixture; + let shared = self.shared.clone(); + let clock = self.clock.clone(); + let capacity = fixture + .get("localMaxSize") + .map(|v| number(Some(v)) as usize) + .unwrap_or(10_000); + let mut builder = DialCache::builder() + .clock_arc(self.clock.clone()) + .runtime_arc(self.exec.runtime.clone()) + .disable_compression() + .local_capacity(capacity) + .shadow_max_in_flight( + fixture + .get("shadowMaxInFlight") + .map(|v| number(Some(v)) as usize) + .unwrap_or(1), + ) + .observer_arc(Arc::new(DriverObserver { + shared: shared.clone(), + clock: clock.clone(), + shadow_hook: fixture.get("shadowHook") != Some(&Value::Bool(false)), + })) + .logger_arc(Arc::new(DriverLogger { + shared: shared.clone(), + })) + .policy_provider({ + let shared = shared.clone(); + move |_identity: Identity| { + let shared = shared.clone(); + async move { + let (index, hold) = { + let mut s = shared.lock(); + let index = s.increment("policyCalls"); + (index, s.fault("holdPolicies")) + }; + if hold { + hold_effect(&shared, "policy", index) + .await + .map_err(|e| -> BoxError { e.into() })?; + } + let s = shared.lock(); + if s.fault("policy") { + return Err("controlled policy failure".into()); + } + Ok(Some(RuntimePolicy::from_json(s.runtime_policy.clone()))) + } + } + }); + if fixture.get("remote") != Some(&Value::Bool(false)) { + builder = builder.remote_arc(Arc::new(DriverRemote { + shared: shared.clone(), + clock: clock.clone(), + })); + } + if fixture.get("localFaultInjection") == Some(&Value::Bool(true)) { + let inner = std::num::NonZeroUsize::new(capacity).map(LruLocalStore::new); + builder = builder.local_store(Box::new(FaultStore { + inner, + shared: shared.clone(), + })); + } + if fixture.get("readTimeoutMs") != Some(&Value::from("default")) { + builder = builder.remote_read_timeout_ms( + fixture + .get("readTimeoutMs") + .map(|v| number(Some(v)) as u64) + .unwrap_or(50), + ); + } + if let Some(mode) = fixture.get("recovery").and_then(Value::as_str) { + if mode != "default" { + builder = builder.should_recover(self.classifier(mode)); + } + } + let cache = builder.build().expect("valid fixture configuration"); + self.instances.insert(id.to_string(), cache.clone()); + cache + } + + fn budget(&self) -> SourceBudget { + match self.fixture.get("fallbackTimeoutMs") { + None => SourceBudget::Millis(10), + Some(Value::Null) => SourceBudget::Unbounded, + Some(Value::String(s)) if s == "default" => SourceBudget::Default, + Some(other) => SourceBudget::Millis(number(Some(other)) as u64), + } + } + + fn scope_of(&self, id: &str) -> Result<(Scope, String), String> { + let handle = self + .scopes + .get(id) + .ok_or_else(|| format!("unknown scope {id}"))?; + let scope = handle + .scope + .lock() + .clone() + .ok_or_else(|| format!("scope {id} not ready"))?; + Ok((scope, handle.instance.clone())) + } + + /// Apply one external command, then settle authorized work. + pub fn apply(&mut self, input: &Value) -> Result<(), String> { + let op = text(input.get("op")); + match op.as_str() { + "begin" => self.begin(input)?, + "resolve" | "reject" => { + let index = number(input.get("loader")); + let mut s = self.shared.lock(); + if index < 0 + || index as usize >= s.loaders.len() + || s.loaders[index as usize].is_settled() + { + return Err(format!("no unsettled source {index}")); + } + let index = index as usize; + let at = self.elapsed_ms(); + let outcome = if op == "reject" { + let gate = s.loaders[index].clone(); + if input.get("error") == Some(&Value::from("timeout")) { + gate.settle(LoaderOutcome::Timeout(index)); + } else { + gate.settle(LoaderOutcome::Fail(index)); + } + "reject" + } else { + let value = match input.get("value") { + None => Val::Absent, + Some(value) => Val::from_json(value.clone()), + }; + let gate = s.loaders[index].clone(); + gate.settle(LoaderOutcome::Value(value)); + "resolve" + }; + s.history.push(HistoryEvent { + event: "sourceSettlement", + id: index, + at, + outcome, + duration_ms: 0.0, + failed: false, + }); + } + "advance" => { + let ms = input.get("ms").and_then(Value::as_f64).unwrap_or(0.0); + if ms < 0.0 { + return Err("negative elapsed advance".to_string()); + } + let deliver = input.get("deliverTimers") != Some(&Value::Bool(false)); + self.exec.advance(ms as i64, deliver); + } + "shiftWall" => self.clock.shift(number(input.get("ms")), true), + "seed" => { + let identity = self.identity(&text(input.get("key")), &text(input.get("useCase"))); + let keys = identity.keys().map_err(|e| e.to_string())?; + let raw = if let Some(frame) = input.get("frameHex").and_then(Value::as_str) { + hex::decode(frame).map_err(|e| e.to_string())? + } else { + let mut encoding = 0u8; + let payload: Vec = + if let Some(h) = input.get("payloadHex").and_then(Value::as_str) { + encoding = 1; + hex::decode(h).map_err(|e| e.to_string())? + } else if let Some(t) = input.get("payloadText").and_then(Value::as_str) { + t.as_bytes().to_vec() + } else if let Some(value) = input.get("value") { + serde_json::to_vec(value).map_err(|e| e.to_string())? + } else { + b"undefined".to_vec() + }; + let stamp = (self.wall_ms() - number(input.get("ageMs"))) as u64; + let mut raw = Vec::with_capacity(10 + payload.len()); + raw.push(1); + raw.extend_from_slice(&stamp.to_be_bytes()); + raw.push(encoding); + raw.extend_from_slice(&payload); + raw + }; + let ttl = input + .get("ttlMs") + .map(|v| number(Some(v))) + .unwrap_or(60_000); + let expires = self.elapsed_ms() + ttl; + self.shared + .lock() + .values + .insert(keys.value, Stored { raw, expires }); + } + "invalidate" => { + let cache = self.instance("default"); + let key = text(input.get("key")); + let key = if key.is_empty() { "1".to_string() } else { key }; + let buffer = number(input.get("futureBufferMs")).max(0) as u64; + let result = self + .exec + .block_on(async move { cache.invalidate("id", &key, buffer).await }); + let status = match result { + Ok(()) => "ok", + Err(Error::Remote(error)) + if error.downcast_ref::().is_some() => + { + "mutation_error" + } + Err(Error::MissingRemote) => "missing_remote", + Err(other) => return Err(format!("unexpected invalidation error: {other}")), + }; + self.shared.lock().push("maintenance", Value::from(status)); + } + "observeMarker" => { + let identity = self.identity(&text(input.get("key")), "").tracked(true); + let keys = identity.keys().map_err(|e| e.to_string())?; + let watermark = keys.watermark.ok_or("tracked identity without watermark")?; + let elapsed = self.elapsed_ms(); + let (cutoff, ttl) = { + let mut s = self.shared.lock(); + match s.raw(&watermark, elapsed) { + Some(raw) => { + let stamp: i64 = String::from_utf8_lossy(&raw) + .parse() + .map_err(|e| format!("{e}"))?; + let ttl = s + .values + .get(&watermark) + .map(|item| item.expires - elapsed) + .unwrap_or(-2); + (stamp - WALL_EPOCH_MS, ttl) + } + None => (-1, -2), + } + }; + let mut fields = Map::new(); + fields.insert("cutoffMs".to_string(), Value::from(cutoff)); + fields.insert("ttlMs".to_string(), Value::from(ttl)); + self.shared.lock().record("marker", fields); + } + "adapterReply" => { + let mut s = self.shared.lock(); + if s.reply.is_some() { + return Err("unconsumed adapter reply".to_string()); + } + s.reply = Some(input.get("value").cloned().unwrap_or(Value::Null)); + } + "policy" => { + self.shared.lock().runtime_policy = + input.get("value").cloned().unwrap_or(Value::Null) + } + "faults" => { + let mut s = self.shared.lock(); + if let Some(Value::Object(faults)) = input.get("value") { + for (k, v) in faults { + s.faults.insert(k.clone(), v.as_bool().unwrap_or(false)); + } + } + } + "release" => { + let effect = text(input.get("effect")); + let index = number(input.get("index")).max(0) as usize; + let gate = { + let mut s = self.shared.lock(); + s.effects + .get_mut(&effect) + .and_then(|gates| gates.remove(&index)) + }; + let Some(gate) = gate.filter(|gate| !gate.is_settled()) else { + return Err(format!("no pending {effect} {index}")); + }; + if input.get("fail") == Some(&Value::Bool(true)) { + gate.settle(Err(format!("controlled {effect} failure"))); + } else { + gate.settle(Ok(())); + } + } + "openScope" => self.open_scope(input)?, + "closeScope" => { + let id = text(input.get("id")); + let handle = self.scopes.get(&id).ok_or("unknown/already closed scope")?; + if handle.gate.is_settled() { + return Err("unknown/already closed scope".to_string()); + } + handle.gate.settle(()); + let done = handle.done.clone(); + self.exec.drain(); + if !done.is_settled() { + return Err(format!("scope {id} did not close")); + } + } + other => return Err(format!("unknown behavior input {other}: {input}")), + } + if self.skip_settle { + let mut s = self.shared.lock(); + let snapshot = s.observed.clone(); + s.unsettled = Some(snapshot); + } + // Drain ready work while unresolved external gates remain held: the + // causally-ready-v1 settlement step. + self.exec.drain(); + Ok(()) + } + + fn begin(&mut self, input: &Value) -> Result<(), String> { + let index = { + let mut s = self.shared.lock(); + let index = s + .observed + .get("calls") + .and_then(Value::as_array) + .map(|c| c.len()) + .unwrap_or(0); + s.push("calls", json!({ "status": "pending" })); + index + }; + let (ctx, instance) = match input.get("scope").and_then(Value::as_str) { + Some(scope_id) => { + let (scope, scope_instance) = self.scope_of(scope_id)?; + let instance = input + .get("instance") + .and_then(Value::as_str) + .map(str::to_string) + .unwrap_or(scope_instance); + (Some(scope), instance) + } + None => ( + None, + input + .get("instance") + .and_then(Value::as_str) + .unwrap_or("default") + .to_string(), + ), + }; + let cache = self.instance(&instance); + let policy = Policy::from_json(self.fixture.get("policy").unwrap_or(&Value::Null)) + .map_err(|e| format!("invalid fixture policy: {e}"))?; + let identity = self.identity(&text(input.get("key")), &text(input.get("useCase"))); + let shared = self.shared.clone(); + let clock = self.clock.clone(); + let comparator_mode = self + .fixture + .get("comparator") + .and_then(Value::as_str) + .map(str::to_string); + let comparison_ms = number(self.fixture.get("comparisonMs")); + let comparator: Box bool + Send + Sync> = match comparator_mode { + None => Box::new(|a: &Val, b: &Val| a == b), + Some(mode) => { + let shared = shared.clone(); + let clock = clock.clone(); + Box::new(move |_a: &Val, _b: &Val| { + shared.lock().increment("comparisons"); + clock.shift(comparison_ms, false); + if mode == "error" { + controlled_panic(); + } + mode == "equal" + }) + } + }; + let codec: Arc> = Arc::new(DriverCodec { + shared: shared.clone(), + }); + let mut operation = Operation::with_codec(identity, codec, comparator) + .policy(policy) + .budget(self.budget()) + .preview(|value: &Val| value.json_preview()); + if let Some(mode) = input.get("recovery").and_then(Value::as_str) { + operation = operation.should_recover(self.classifier(mode)); + } + let probe_scope = self.fixture.get("probeSourceScope") == Some(&Value::Bool(true)); + let source_work_ms = number(self.fixture.get("sourceWorkMs")); + let source_cache = cache.clone(); + let source_shared = shared.clone(); + let source_clock = clock.clone(); + let load = move |scope: Scope| { + let shared = source_shared.clone(); + let clock = source_clock.clone(); + let cache = source_cache.clone(); + async move { + let gate = { + let mut s = shared.lock(); + let id = s.loaders.len(); + let gate: Gate = Gate::new(); + s.loaders.push(gate.clone()); + s.increment("loaders"); + use dialcache::Clock; + let at = clock.elapsed_ms(); + s.history.push(HistoryEvent { + event: "sourceStart", + id, + at, + outcome: "", + duration_ms: 0.0, + failed: false, + }); + if probe_scope { + let enabled = cache.is_enabled(&scope); + s.push("sourceScopes", Value::Bool(enabled)); + } + gate + }; + clock.shift(source_work_ms, false); + match gate.wait().await { + LoaderOutcome::Value(value) => Ok(value), + LoaderOutcome::Fail(id) => Err(Box::new(SourceFailure(id)) as BoxError), + LoaderOutcome::Timeout(id) => { + // A timeout propagated by the source keeps the loader's identity. + Err(Box::new(FallbackTimeout { + use_case: format!("NestedSource:{id}"), + timeout_ms: 10, + }) as BoxError) + } + } + } + }; + let call_shared = shared.clone(); + let call_cache = cache.clone(); + let call = move |scope: Scope| { + let shared = call_shared.clone(); + let cache = call_cache.clone(); + let operation = operation.clone(); + let load = load.clone(); + async move { + let result = cache.get_or_load(&scope, operation, load).await; + let mut s = shared.lock(); + let record = match result { + Ok(value) => json!({ "status": "value", "value": value.observed() }), + Err(error) => json!({ "status": "error", "error": s.classify_error(&error) }), + }; + if let Some(Value::Array(calls)) = s.observed.get_mut("calls") { + calls[index] = record; + } + } + }; + let disabled = input.get("disabled") == Some(&Value::Bool(true)); + let outside = input.get("outside") == Some(&Value::Bool(true)); + match ctx { + Some(ctx) => { + if disabled { + self.exec + .spawn(async move { cache.disable_in(&ctx, call).await }); + } else { + self.exec.spawn(call(ctx)); + } + } + None if outside => { + let ctx = Scope::outside(); + if disabled { + self.exec + .spawn(async move { cache.disable_in(&ctx, call).await }); + } else { + self.exec.spawn(call(ctx)); + } + } + None => { + if disabled { + let inner = cache.clone(); + self.exec.spawn(async move { + cache + .enable(|scope| async move { inner.disable_in(&scope, call).await }) + .await + }); + } else { + self.exec.spawn(async move { cache.enable(call).await }); + } + } + } + Ok(()) + } + + fn open_scope(&mut self, input: &Value) -> Result<(), String> { + let id = text(input.get("id")); + if self.scopes.contains_key(&id) { + return Err(format!("duplicate scope {id}")); + } + let parent = match input.get("parent").and_then(Value::as_str) { + Some(parent_id) => Some(self.scope_of(parent_id)?), + None => None, + }; + let instance = input + .get("instance") + .and_then(Value::as_str) + .map(str::to_string) + .or_else(|| parent.as_ref().map(|(_, instance)| instance.clone())) + .unwrap_or_else(|| "default".to_string()); + let cache = self.instance(&instance); + let handle = ScopeHandle { + instance, + scope: Arc::new(Mutex::new(None)), + gate: Gate::new(), + done: Gate::new(), + }; + let slot = handle.scope.clone(); + let gate = handle.gate.clone(); + let done = handle.done.clone(); + let disabled = input.get("disabled") == Some(&Value::Bool(true)); + let body = move |scope: Scope| { + let slot = slot.clone(); + let gate = gate.clone(); + async move { + *slot.lock() = Some(scope); + gate.wait().await; + } + }; + let parent_scope = parent.map(|(scope, _)| scope); + self.exec.spawn(async move { + match (parent_scope, disabled) { + (Some(parent), true) => cache.disable_in(&parent, body).await, + (Some(parent), false) => cache.enable_in(&parent, body).await, + (None, true) => cache.disable_in(&Scope::outside(), body).await, + (None, false) => cache.enable(body).await, + } + done.settle(()); + }); + self.exec.drain(); + if handle.scope.lock().is_none() { + return Err(format!("scope {id} did not open")); + } + self.scopes.insert(id, handle); + Ok(()) + } + + /// The complete current observation. + pub fn observation(&self) -> Value { + let s = self.shared.lock(); + if self.skip_settle { + if let Some(unsettled) = &s.unsettled { + return Value::Object(unsettled.clone()); + } + } + Value::Object(s.observed.clone()) + } + + /// Release every driver-owned gate so no work leaks into the next history. + pub fn close(&mut self) { + { + let mut s = self.shared.lock(); + for name in [ + "holdReads", + "holdWrites", + "holdDumps", + "holdLoads", + "holdPolicies", + ] { + s.faults.insert(name.to_string(), false); + } + } + for handle in self.scopes.values() { + handle.gate.settle(()); + } + loop { + let mut pending = false; + { + let mut s = self.shared.lock(); + for gates in s.effects.values_mut() { + for gate in gates.values() { + if gate.settle(Ok(())) { + pending = true; + } + } + } + for gate in &s.loaders { + if gate.settle(LoaderOutcome::Value(Val::Json(json!(0)))) { + pending = true; + } + } + } + self.exec.drain(); + if !pending { + break; + } + } + } + + pub fn history(&self) -> Vec { + self.shared.lock().history.clone() + } + + /// C23/C25/C26 monitor over the actual callback history (effects profile). + pub fn assert_effects_history(&self) -> Result<(), String> { + struct Source { + at: i64, + settled: &'static str, + } + let history = self.history(); + let mut sources: HashMap = HashMap::new(); + let mut active: Option = None; + let mut authorized = false; + let mut previous = -1i64; + for (index, e) in history.iter().enumerate() { + let fail = |reason: &str| { + Err(format!( + "effects contract event {index} {}: {reason}", + e.event + )) + }; + if e.at < previous { + return fail("elapsed time moved backward"); + } + previous = e.at; + match e.event { + "sourceStart" => { + if sources.contains_key(&e.id) || active.is_some() { + return fail("source started before prior fallback completed"); + } + sources.insert( + e.id, + Source { + at: e.at, + settled: "", + }, + ); + active = Some(e.id); + authorized = false; + } + "sourceSettlement" => { + let Some(source) = sources.get_mut(&e.id) else { + return fail("invalid source settlement identity"); + }; + if !source.settled.is_empty() + || (e.outcome != "resolve" && e.outcome != "reject") + { + return fail("invalid source settlement identity"); + } + source.settled = e.outcome; + } + "fallbackCompletion" => { + let Some(id) = active else { + return fail("fallback completion has no source"); + }; + let source = &sources[&id]; + if !e.duration_ms.is_finite() || e.duration_ms < 0.0 { + return fail("invalid observed fallback duration"); + } + let elapsed = (e.at - source.at) as f64; + if (e.duration_ms - elapsed).abs() > 1e-7 { + return Err(format!("C23: duration includes lookup or omits source time (event {index}, at {} elapsed {elapsed} duration {})", e.at, e.duration_ms)); + } + if !e.failed && (elapsed >= 10.0 || source.settled != "resolve") { + return Err(format!("C25: success must be accepted before its source deadline (event {index}, elapsed {elapsed}, settlement {})", source.settled)); + } + if e.failed && elapsed < 10.0 && source.settled != "reject" { + return Err(format!("C23: source lost its full source-relative budget (event {index}, elapsed {elapsed}, settlement {})", source.settled)); + } + active = None; + authorized = !e.failed; + } + "writeDispatch" => { + if !authorized { + return Err(format!( + "C26: publication without accepted source success (event {index})" + )); + } + } + _ => return fail("unknown monitor event"), + } + } + Ok(()) + } +} + +/// Block an external effect until the history releases it. +fn hold_effect( + shared: &Arc>, + effect: &str, + index: usize, +) -> impl std::future::Future> + Send { + let gate: Gate> = Gate::new(); + shared + .lock() + .effects + .entry(effect.to_string()) + .or_default() + .insert(index, gate.clone()); + async move { gate.wait().await } +} + +struct DriverCodec { + shared: Arc>, +} + +impl Codec for DriverCodec { + fn encode<'a>(&'a self, value: &'a Val) -> BoxFuture<'a, Result> { + let shared = self.shared.clone(); + let value = value.clone(); + Box::pin(async move { + let (index, hold) = { + let mut s = shared.lock(); + let index = s.increment("dumps"); + (index, s.fault("holdDumps")) + }; + if hold { + hold_effect(&shared, "dump", index) + .await + .map_err(|e| -> BoxError { e.into() })?; + } + if shared.lock().fault("dump") { + return Err("controlled serialization failure".into()); + } + Ok(match value { + Val::Absent => Payload::text("undefined"), + Val::Json(json) => Payload::text(serde_json::to_string(&json)?), + }) + }) + } + + fn decode(&self, payload: Payload) -> BoxFuture<'_, Result> { + let shared = self.shared.clone(); + Box::pin(async move { + let (index, hold) = { + let mut s = shared.lock(); + let index = s.increment("loads"); + (index, s.fault("holdLoads")) + }; + if hold { + hold_effect(&shared, "load", index) + .await + .map_err(|e| -> BoxError { e.into() })?; + } + if shared.lock().fault("load") { + return Err("controlled deserialization failure".into()); + } + if payload.bytes == b"undefined" { + return Ok(Val::Absent); + } + let value: Value = serde_json::from_slice(&payload.bytes)?; + Ok(Val::Json(value)) + }) + } +} + +struct DriverRemote { + shared: Arc>, + clock: Arc, +} + +impl Remote for DriverRemote { + fn read( + &self, + request: ReadRequest, + context: ReadContext, + ) -> BoxFuture<'_, Result> { + let shared = self.shared.clone(); + let clock = self.clock.clone(); + Box::pin(async move { + let (index, hold) = { + let mut s = shared.lock(); + let index = s.increment("reads"); + let mut fields = Map::new(); + fields.insert("index".to_string(), Value::from(index)); + fields.insert("timeoutMs".to_string(), Value::from(context.timeout_ms)); + fields.insert( + "aborted".to_string(), + Value::Bool(context.cancel.is_cancelled()), + ); + s.record("readContext", fields); + (index, s.fault("holdReads")) + }; + { + let shared = shared.clone(); + context.cancel.on_cancel(move || { + let mut fields = Map::new(); + fields.insert("index".to_string(), Value::from(index)); + shared.lock().record("readAbort", fields); + }); + } + if hold { + hold_effect(&shared, "read", index) + .await + .map_err(|e| -> BoxError { e.into() })?; + } + let mut s = shared.lock(); + if s.fault("read") { + return Err("controlled read failure".into()); + } + if let Some(reply) = s.reply.take() { + return Ok(read_result_from_untrusted_json(&reply)); + } + use dialcache::Clock; + let elapsed = clock.elapsed_ms(); + let raw = s.raw(&request.value_key, elapsed); + let marker = match &request.watermark_key { + Some(key) => s.raw(key, elapsed), + None => None, + }; + let tracked = request.watermark_key.is_some(); + Ok(decode_frame(raw.as_deref(), tracked, marker.as_deref())?) + }) + } + + fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + let shared = self.shared.clone(); + let clock = self.clock.clone(); + Box::pin(async move { + use dialcache::Clock; + let (index, hold) = { + let mut s = shared.lock(); + let index = s.increment("writes"); + let at = clock.elapsed_ms(); + s.history.push(HistoryEvent { + event: "writeDispatch", + id: 0, + at, + outcome: "", + duration_ms: 0.0, + failed: false, + }); + let mut fields = Map::new(); + fields.insert("index".to_string(), Value::from(index)); + s.record("writeDispatch", fields); + s.push("writeTtls", Value::from(request.ttl_ms)); + (index, s.fault("holdWrites")) + }; + if hold { + hold_effect(&shared, "write", index) + .await + .map_err(|e| -> BoxError { e.into() })?; + } + let mut s = shared.lock(); + if s.fault("write") { + return Err(Box::new(MaintenanceFailure) as BoxError); + } + let raw = encode_frame(&request.frame)?; + if !s.discard_writes { + let expires = clock.elapsed_ms() + request.ttl_ms as i64; + s.values.insert(request.value_key, Stored { raw, expires }); + } + Ok(()) + }) + } + + fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + let shared = self.shared.clone(); + let clock = self.clock.clone(); + Box::pin(async move { + use dialcache::Clock; + let mut s = shared.lock(); + s.increment("invalidations"); + if s.fault("write") { + return Err(Box::new(MaintenanceFailure) as BoxError); + } + if s.discard_invalidations { + return Ok(()); + } + let now = request.invalidated_at_ms as i64; + let mut cutoff = now + request.future_buffer_ms as i64; + let elapsed = clock.elapsed_ms(); + let prior = s.raw(&request.watermark_key, elapsed); + let old: i64 = prior + .and_then(|raw| String::from_utf8_lossy(&raw).parse().ok()) + .unwrap_or(0); + if old > cutoff { + cutoff = old; + } + let mut ttl: i64 = 7_200_000; + if cutoff - now + 3_660_000 > ttl { + ttl = cutoff - now + 3_660_000; + } + if let Some(item) = s.values.get(&request.watermark_key) { + if item.expires - elapsed > ttl { + ttl = item.expires - elapsed; + } + } + s.values.insert( + request.watermark_key, + Stored { + raw: cutoff.to_string().into_bytes(), + expires: elapsed + ttl, + }, + ); + Ok(()) + }) + } +} + +struct DriverObserver { + shared: Arc>, + clock: Arc, + shadow_hook: bool, +} + +fn labels_map(labels: &dialcache::Labels) -> Map { + let mut m = Map::new(); + m.insert( + "cacheNamespace".to_string(), + Value::from(labels.namespace.as_ref()), + ); + m.insert("useCase".to_string(), Value::from(labels.use_case.as_ref())); + m.insert("keyType".to_string(), Value::from(labels.key_type.as_ref())); + m.insert("layer".to_string(), Value::from(labels.layer.as_str())); + m +} + +fn outcome_map(labels: &dialcache::observe::OutcomeLabels) -> Map { + let mut m = Map::new(); + m.insert( + "cacheNamespace".to_string(), + Value::from(labels.namespace.as_ref()), + ); + m.insert("useCase".to_string(), Value::from(labels.use_case.as_ref())); + m.insert("keyType".to_string(), Value::from(labels.key_type.as_ref())); + m +} + +impl Observer for DriverObserver { + fn observe(&self, event: &Event) { + use dialcache::Clock; + let mut s = self.shared.lock(); + let (kind, mut fields): (&str, Map) = match event { + Event::Request { labels } => ("request", labels_map(labels)), + Event::Miss { labels, reason } => { + let mut m = labels_map(labels); + m.insert("reason".to_string(), Value::from(reason.as_str())); + ("miss", m) + } + Event::Disabled { labels, reason } => { + let mut m = labels_map(labels); + m.insert("reason".to_string(), Value::from(reason.as_str())); + ("disabled", m) + } + Event::Error { + labels, + error, + in_fallback, + } => { + let mut m = labels_map(labels); + m.insert("error".to_string(), Value::from(error.as_str())); + m.insert("inFallback".to_string(), Value::Bool(*in_fallback)); + if *in_fallback && *error == dialcache::observe::ErrorKind::Fallback { + s.fallback_failed = true; + } + ("error", m) + } + Event::Invalidation { + namespace, + key_type, + layer, + } => { + let mut m = Map::new(); + m.insert( + "cacheNamespace".to_string(), + Value::from(namespace.as_ref()), + ); + m.insert("keyType".to_string(), Value::from(key_type.as_ref())); + m.insert("layer".to_string(), Value::from(layer.as_str())); + ("invalidation", m) + } + Event::Coalesced { labels, scope } => { + let mut m = outcome_map(labels); + m.insert("scope".to_string(), Value::from(scope.as_str())); + ("coalesced", m) + } + Event::ShadowValidation { labels, outcome } => { + s.push("shadow", Value::from(outcome.as_str())); + let mut m = outcome_map(labels); + m.insert("outcome".to_string(), Value::from(outcome.as_str())); + ("shadowValidation", m) + } + Event::ShadowValueAge { + labels, + outcome, + seconds, + } => { + let mut m = outcome_map(labels); + m.insert("outcome".to_string(), Value::from(outcome.as_str())); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("shadowAge", m) + } + Event::FutureTimestampOffset { labels, seconds } => { + let mut m = labels_map(labels); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("futureOffset", m) + } + Event::StaleRecovery { labels, outcome } => { + s.push("recovery", Value::from(outcome.as_str())); + let mut m = outcome_map(labels); + m.insert("outcome".to_string(), Value::from(outcome.as_str())); + ("staleRecovery", m) + } + Event::StaleRecoveryValueAge { + labels, + outcome, + seconds, + } => { + let mut m = outcome_map(labels); + m.insert("outcome".to_string(), Value::from(outcome.as_str())); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("recoveryAge", m) + } + Event::Compression { labels, outcome } => { + let mut m = labels_map(labels); + m.insert("outcome".to_string(), Value::from(outcome.as_str())); + ("compression", m) + } + Event::Get { labels, seconds } => { + let mut m = labels_map(labels); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("get", m) + } + Event::Fallback { labels, seconds } => { + let mut m = labels_map(labels); + m.insert("seconds".to_string(), Value::from(*seconds)); + let at = self.clock.elapsed_ms(); + let failed = s.fallback_failed; + s.history.push(HistoryEvent { + event: "fallbackCompletion", + id: 0, + at, + outcome: "", + duration_ms: seconds * 1000.0, + failed, + }); + s.fallback_failed = false; + ("fallback", m) + } + Event::Serialization { + labels, + operation, + seconds, + } => { + let mut m = labels_map(labels); + m.insert("operation".to_string(), Value::from(operation.as_str())); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("serialization", m) + } + Event::Size { labels, bytes } => { + let mut m = labels_map(labels); + m.insert("bytes".to_string(), Value::from(*bytes)); + ("size", m) + } + Event::StoredSize { labels, bytes } => { + let mut m = labels_map(labels); + m.insert("bytes".to_string(), Value::from(*bytes)); + ("storedSize", m) + } + Event::CompressionRatio { labels, ratio } => { + let mut m = labels_map(labels); + m.insert("value".to_string(), Value::from(*ratio)); + ("compressionRatio", m) + } + Event::CompressionDuration { + labels, + operation, + seconds, + } => { + let mut m = labels_map(labels); + m.insert("operation".to_string(), Value::from(operation.as_str())); + m.insert("seconds".to_string(), Value::from(*seconds)); + ("compressionDuration", m) + } + }; + fields.retain(|k, _| k != "value" || kind == "compressionRatio"); + s.record(kind, fields); + let fail = s.observer_fails(); + drop(s); + if fail { + controlled_panic(); + } + } + + fn observes_shadow_outcomes(&self) -> bool { + self.shadow_hook + } +} + +struct DriverLogger { + shared: Arc>, +} + +impl Logger for DriverLogger { + fn log(&self, event: &LogEvent) { + let mut s = self.shared.lock(); + if let LogEvent::ShadowMismatch(details) = event { + let mut fields = Map::new(); + fields.insert( + "cacheNamespace".to_string(), + Value::from(details.namespace.as_ref()), + ); + fields.insert( + "useCase".to_string(), + Value::from(details.use_case.as_ref()), + ); + fields.insert( + "keyType".to_string(), + Value::from(details.key_type.as_ref()), + ); + fields.insert("outcome".to_string(), Value::from("mismatch")); + fields.insert( + "cacheKey".to_string(), + Value::from(details.cache_key.as_str()), + ); + fields.insert( + "cachedValueJson".to_string(), + details + .cached_value_json + .clone() + .map(Value::from) + .unwrap_or(Value::Null), + ); + fields.insert( + "sourceValueJson".to_string(), + details + .source_value_json + .clone() + .map(Value::from) + .unwrap_or(Value::Null), + ); + s.record("mismatchWarning", fields); + } + let fail = s.observer_fails(); + drop(s); + if fail { + controlled_panic(); + } + } +} + +/// Local storage that fails on command, at the native storage boundary. +struct FaultStore { + inner: Option, + shared: Arc>, +} + +impl LocalStore for FaultStore { + fn get(&mut self, key: &str, now_ms: i64) -> Result, BoxError> { + if self.shared.lock().fault("localStorage") { + return Err("controlled local storage failure".into()); + } + match self.inner.as_mut() { + Some(inner) => inner.get(key, now_ms), + None => Ok(None), + } + } + + fn put(&mut self, key: String, entry: LocalEntry) -> Result<(), BoxError> { + if self.shared.lock().fault("localStorage") { + return Err("controlled local storage failure".into()); + } + match self.inner.as_mut() { + Some(inner) => inner.put(key, entry), + None => Ok(()), + } + } +} + +#[allow(dead_code)] +fn _unused(shared: &Shared) -> Map { + shared_labels(shared) +} diff --git a/rust/tests/formal/gate.rs b/rust/tests/formal/gate.rs new file mode 100644 index 00000000..0f499856 --- /dev/null +++ b/rust/tests/formal/gate.rs @@ -0,0 +1,88 @@ +//! A settle-once gate the driver holds and library callbacks await. + +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll, Waker}; + +use parking_lot::Mutex; + +struct GateState { + value: Option, + wakers: Vec, +} + +/// Driver-owned gate: external work blocks on it until the history releases it. +pub struct Gate { + inner: Arc>>, +} + +impl Clone for Gate { + fn clone(&self) -> Self { + Gate { + inner: self.inner.clone(), + } + } +} + +impl Default for Gate { + fn default() -> Self { + Gate::new() + } +} + +impl Gate { + pub fn new() -> Self { + Gate { + inner: Arc::new(Mutex::new(GateState { + value: None, + wakers: Vec::new(), + })), + } + } + + /// Release every waiter with `value`. Returns false if already settled. + pub fn settle(&self, value: T) -> bool { + let wakers = { + let mut state = self.inner.lock(); + if state.value.is_some() { + return false; + } + state.value = Some(value); + std::mem::take(&mut state.wakers) + }; + for waker in wakers { + waker.wake(); + } + true + } + + pub fn is_settled(&self) -> bool { + self.inner.lock().value.is_some() + } + + pub fn wait(&self) -> GateWait { + GateWait { gate: self.clone() } + } +} + +pub struct GateWait { + gate: Gate, +} + +impl Unpin for GateWait {} + +impl Future for GateWait { + type Output = T; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let mut state = self.gate.inner.lock(); + if let Some(value) = &state.value { + return Poll::Ready(value.clone()); + } + if !state.wakers.iter().any(|w| w.will_wake(cx.waker())) { + state.wakers.push(cx.waker().clone()); + } + Poll::Pending + } +} diff --git a/rust/tests/formal/local_clock.rs b/rust/tests/formal/local_clock.rs new file mode 100644 index 00000000..63e853c6 --- /dev/null +++ b/rust/tests/formal/local_clock.rs @@ -0,0 +1,122 @@ +//! The local-clock profile driver: default-aligned instances over a shared +//! virtual grid, fractional environment ticks, immediate healthy sources. + +use std::sync::Arc; + +use dialcache::testing::{TestExecutor, VirtualGridClock}; +use dialcache::{DialCache, Identity, Operation, Policy}; +use parking_lot::Mutex; +use serde_json::{json, Map, Value}; + +use super::driver::WALL_EPOCH_MS; + +pub struct LocalClockDriver { + pub exec: TestExecutor, + caches: [Option; 2], + sources: Arc>, + calls: Vec, +} + +impl std::fmt::Debug for LocalClockDriver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("LocalClockDriver").finish() + } +} + +impl LocalClockDriver { + pub fn new() -> LocalClockDriver { + LocalClockDriver { + exec: TestExecutor::new(WALL_EPOCH_MS), + caches: [None, None], + sources: Arc::new(Mutex::new(0)), + calls: Vec::new(), + } + } + + pub fn apply(&mut self, input: &Value) -> Result<(), String> { + let instance = input.get("instance").and_then(Value::as_i64).unwrap_or(0); + match input.get("op").and_then(Value::as_str).unwrap_or("") { + "constructInstance" => { + if !(0..2).contains(&instance) || self.caches[instance as usize].is_some() { + return Err("invalid/duplicate instance".to_string()); + } + let cache = DialCache::builder() + .clock(VirtualGridClock::new(self.exec.clock.clone())) + .runtime_arc(self.exec.runtime.clone()) + .build() + .map_err(|e| e.to_string())?; + self.caches[instance as usize] = Some(cache); + Ok(()) + } + "advanceTicks" => { + let ticks = input.get("ticks").and_then(Value::as_i64).unwrap_or(0); + if ticks <= 0 { + return Err("invalid clock advance".to_string()); + } + self.exec.advance_micros(ticks); + Ok(()) + } + "call" => { + if !(0..2).contains(&instance) { + return Err("call before instance construction".to_string()); + } + let Some(cache) = self.caches[instance as usize].clone() else { + return Err("call before instance construction".to_string()); + }; + let offered = input.get("offered").and_then(Value::as_i64).unwrap_or(0); + let sources = self.sources.clone(); + let operation = + Operation::::new(Identity::new("clock", "one", "QuintLocalGrid")) + .policy(Policy::default().local_ttl_sec(1)); + let value = self.exec.block_on(async move { + let inner = cache.clone(); + cache + .enable(|scope| async move { + inner + .get_or_load(&scope, operation, move |_| { + let sources = sources.clone(); + async move { + *sources.lock() += 1; + Ok(offered) + } + }) + .await + }) + .await + }); + let value = value.map_err(|e| e.to_string())?; + self.calls.push(*value); + Ok(()) + } + other => Err(format!("unknown local-clock command {other}")), + } + } + + pub fn observation(&self) -> Value { + let mut o = Map::new(); + o.insert("calls".to_string(), json!(self.calls)); + o.insert("loaders".to_string(), json!(*self.sources.lock())); + for key in [ + "reads", + "writes", + "invalidations", + "loads", + "dumps", + "policyCalls", + "classifications", + "comparisons", + ] { + o.insert(key.to_string(), json!(0)); + } + for key in [ + "maintenance", + "sourceScopes", + "writeTtls", + "shadow", + "recovery", + ] { + o.insert(key.to_string(), json!([])); + } + Value::Object(o) + } +} diff --git a/rust/tests/formal/mod.rs b/rust/tests/formal/mod.rs index 5f08ebf3..f4820325 100644 --- a/rust/tests/formal/mod.rs +++ b/rust/tests/formal/mod.rs @@ -8,9 +8,14 @@ //! per binary. #![allow(dead_code)] +pub mod core_driver; +pub mod driver; +pub mod gate; pub mod inventory; pub mod json; +pub mod local_clock; pub mod report; +pub mod scenarios; pub mod schema; pub mod transport; pub mod witness; diff --git a/rust/tests/formal/scenarios.rs b/rust/tests/formal/scenarios.rs new file mode 100644 index 00000000..01f01624 --- /dev/null +++ b/rust/tests/formal/scenarios.rs @@ -0,0 +1,88 @@ +//! Fixed behavioral scenarios: handwritten command/expectation sequences +//! replayed through the behavior driver. + +use serde_json::{Map, Value}; + +use super::driver::{empty_observation, Driver}; +use super::inventory::repo_path; +use super::json::{json_equal, strict_parse}; + +/// Load and validate `formal/behavioral-scenarios.json`. +pub fn load_scenarios() -> Result, String> { + let text = std::fs::read_to_string(repo_path("formal/behavioral-scenarios.json")) + .map_err(|e| e.to_string())?; + let corpus = strict_parse(&text)?; + if corpus.get("schemaVersion").and_then(Value::as_i64) != Some(2) { + return Err("unsupported behavioral schema".to_string()); + } + let scenarios = corpus + .get("scenarios") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + if scenarios.is_empty() { + return Err("empty behavioral corpus".to_string()); + } + let mut seen = std::collections::HashSet::new(); + let shape = empty_observation(&serde_json::json!({ "observe": [] })); + for scenario in &scenarios { + let name = scenario.get("name").and_then(Value::as_str).unwrap_or(""); + let steps = scenario + .get("steps") + .and_then(Value::as_array) + .cloned() + .unwrap_or_default(); + if name.is_empty() || !seen.insert(name.to_string()) || steps.is_empty() { + return Err(format!("invalid/duplicate/empty scenario {name:?}")); + } + for step in &steps { + if !step.get("input").is_some_and(Value::is_object) { + return Err(format!("missing input in {name}")); + } + let Some(patch) = step.get("expect").and_then(Value::as_object) else { + return Err(format!("missing expected patch in {name}")); + }; + for key in patch.keys() { + if !shape.contains_key(key) { + return Err(format!("unknown expected field {key}")); + } + } + } + } + Ok(scenarios) +} + +/// Replay one scenario; every step's observation must equal the accumulated expectation. +pub fn replay_scenario(scenario: &Value) -> Result<(), String> { + let fixture = scenario.get("fixture").cloned().unwrap_or(Value::Null); + let name = scenario.get("name").and_then(Value::as_str).unwrap_or(""); + let mut driver = Driver::new(fixture.clone()); + let mut expected: Map = empty_observation(&fixture); + let result = (|| { + for (index, step) in scenario + .get("steps") + .and_then(Value::as_array) + .into_iter() + .flatten() + .enumerate() + { + if let Some(patch) = step.get("expect").and_then(Value::as_object) { + for (key, value) in patch { + expected.insert(key.clone(), value.clone()); + } + } + let input = step.get("input").cloned().unwrap_or(Value::Null); + driver + .apply(&input) + .map_err(|e| format!("{name} step {index} input {input}: {e}"))?; + let actual = driver.observation(); + let expected_value = Value::Object(expected.clone()); + if !json_equal(&expected_value, &actual) { + return Err(format!("{name} step {index} input {input}\nexpected: {expected_value}\nactual: {actual}")); + } + } + Ok(()) + })(); + driver.close(); + result +} diff --git a/rust/tests/settlement_control.rs b/rust/tests/settlement_control.rs new file mode 100644 index 00000000..7ff83cf3 --- /dev/null +++ b/rust/tests/settlement_control.rs @@ -0,0 +1,74 @@ +//! The no-settle control: a driver that reports the observation held before +//! the settlement drain must fail the observation assertions of every +//! behavior-driver-backed smoke history. It proves the replays depend on the +//! `causally-ready-v1` settlement. + +mod formal; + +use formal::driver::{install_panic_hook, Driver}; +use formal::inventory::repo_path; +use formal::transport::Coordinator; +use serde_json::Value; + +const BEHAVIOR_SMOKE: [&str; 14] = [ + "effects", + "admission", + "independent", + "layers", + "local-failure", + "policy", + "recovery", + "recovery-read", + "runtime-boundaries", + "scope", + "shadow", + "shadow-layers", + "source-budgets", + "effects", +]; + +fn replay(coordinator: &mut Coordinator, profile: &str, skip_settle: bool) -> Result<(), String> { + let path = repo_path(&format!( + "formal/{}-smoke.itf.json", + if profile == "effects" { + "effects" + } else { + profile + } + )); + let prepared = coordinator.prepare(profile, &path, None)?; + let mut driver = Driver::new(prepared.fixture.clone()); + driver.skip_settle = skip_settle; + let result = { + let cell = std::cell::RefCell::new(&mut driver); + let mut apply = |input: &Value| cell.borrow_mut().apply(input); + let mut observation = || cell.borrow().observation(); + let mut wall = || cell.borrow().wall_ms(); + coordinator.execute(&prepared, &mut apply, &mut observation, &mut wall, &mut []) + }; + driver.close(); + result +} + +#[test] +fn unsettled_observations_fail_every_behavior_smoke_history() { + install_panic_hook(); + let mut coordinator = Coordinator::spawn().expect("coordinator"); + let mut profiles: Vec<&str> = BEHAVIOR_SMOKE.to_vec(); + profiles.dedup(); + for profile in profiles { + replay(&mut coordinator, profile, false) + .unwrap_or_else(|e| panic!("{profile} settled replay must pass: {e}")); + let unsettled = replay(&mut coordinator, profile, true); + assert!( + unsettled.is_err(), + "{profile}: an unsettled observation was accepted" + ); + let message = unsettled.unwrap_err(); + assert!( + message.contains("expected:") && message.contains("actual:"), + "{profile}: control failed without comparison evidence: {message}" + ); + } + coordinator.finish().expect("coordinator exit"); +} From edc6dbf7ebf018f64d08e30f91fb40d1a7c5d732 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 00:59:33 -0700 Subject: [PATCH 11/31] test(rust): satisfy clippy in the conformance harness --- rust/tests/formal/driver.rs | 9 ++++----- rust/tests/formal/fixtures.rs | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/rust/tests/formal/driver.rs b/rust/tests/formal/driver.rs index 1e83ba4d..ffebae96 100644 --- a/rust/tests/formal/driver.rs +++ b/rust/tests/formal/driver.rs @@ -43,6 +43,8 @@ pub fn install_panic_hook() { })); } +type ComparatorFn = Box bool + Send + Sync>; + /// The N-th controlled source failure. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] #[error("source failure {0}")] @@ -176,10 +178,7 @@ impl Shared { } fn raw(&mut self, key: &str, elapsed_ms: i64) -> Option> { - let expired = match self.values.get(key) { - None => return None, - Some(item) => item.expires <= elapsed_ms, - }; + let expired = self.values.get(key)?.expires <= elapsed_ms; if expired { self.values.remove(key); return None; @@ -722,7 +721,7 @@ impl Driver { .and_then(Value::as_str) .map(str::to_string); let comparison_ms = number(self.fixture.get("comparisonMs")); - let comparator: Box bool + Send + Sync> = match comparator_mode { + let comparator: ComparatorFn = match comparator_mode { None => Box::new(|a: &Val, b: &Val| a == b), Some(mode) => { let shared = shared.clone(); diff --git a/rust/tests/formal/fixtures.rs b/rust/tests/formal/fixtures.rs index 9489981d..7e4f2c26 100644 --- a/rust/tests/formal/fixtures.rs +++ b/rust/tests/formal/fixtures.rs @@ -343,6 +343,7 @@ pub fn protocol_groups(selection: &str) -> BTreeMap> { #[cfg(test)] mod tests { + #[allow(unused_imports)] use super::*; #[test] From 530dcd2c045b83555e933c6d236700d9f8c1d5c1 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 01:03:11 -0700 Subject: [PATCH 12/31] docs(rust): add the Rust guide and link it from the README --- README.md | 6 +- formal/go-parity.json | 6 +- formal/source-audit.json | 2 +- rust/README.md | 167 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 7 deletions(-) create mode 100644 rust/README.md diff --git a/README.md b/README.md index 2e14012e..68b83bd4 100644 --- a/README.md +++ b/README.md @@ -132,10 +132,10 @@ for sampling and comparison behavior. | Recovery from selected source failures | [Stale-on-error](https://lan17.github.io/DialCache/stale-on-error.html) | | Shared execution and deadlines | [Coalescing and liveness](https://lan17.github.io/DialCache/coalescing.html) | | Methods, options, and exports | [API reference](https://lan17.github.io/DialCache/api.html) | -| Go implementation and shared behavior contracts | [Go guide](go/README.md) · [Quint specification](formal/README.md) · [Worked walkthrough](formal/WALKTHROUGH.md) | +| Go and Rust implementations and shared behavior contracts | [Go guide](go/README.md) · [Rust guide](rust/README.md) · [Quint specification](formal/README.md) · [Worked walkthrough](formal/WALKTHROUGH.md) | -The Go port and the TypeScript library replay the same Quint-generated -histories. Whether those histories reach every required boundary is decided by +The Go port, the Rust port and the TypeScript library replay the same +Quint-generated histories. Whether those histories reach every required boundary is decided by one language-neutral evaluator, `node formal/witnesses.mjs evaluate`, that any port runs over the same corpus; no port depends on another port's test suite for its completion evidence. diff --git a/formal/go-parity.json b/formal/go-parity.json index ea89824e..05088110 100644 --- a/formal/go-parity.json +++ b/formal/go-parity.json @@ -10,7 +10,7 @@ "inputs": { "semanticCasesSha256": "7054107347f57a452e17de99f74310553b7af4350363c674e70ff8184d0610be", "executionSha256": "feee09a3487deb2694f8784b7e8d2d0ee28050dae03ac43ac2613608ce5ab1c6", - "sourceAuditSha256": "4bd6d53038723b133a52e17303342c5a7f46c395f250073d2e1c55fd1d87ea33", + "sourceAuditSha256": "09e9981795420d50cbd30d7f56ef4aeec750724961e794c2dd5526199dc91446", "quintCaseAuditSha256": "dcb576948a68a066e43259d4bee2b3cdadbe82e657ae9af99e4b21ed7b95dbb3", "featureCoverageSha256": "56eedfa865682db6ad8e73b8673c7dac1c30d301e9bce3390246d6994aba9dcb", "coverageWitnessesSha256": "8fc8ab123150fdec059f5b75f84172d39af80d08d7a4211fa88c63c5a3f2874b", @@ -18305,14 +18305,14 @@ ], "reviewedTestAndDocumentationAudit": { "path": "formal/source-audit.json", - "sha256": "4bd6d53038723b133a52e17303342c5a7f46c395f250073d2e1c55fd1d87ea33", + "sha256": "09e9981795420d50cbd30d7f56ef4aeec750724961e794c2dd5526199dc91446", "sourceFiles": 44, "status": "reviewed-applicability-not-execution", "limitations": "These rows review Go applicability for the public documentation and ordinary test files in source-audit.json sources, inheriting their C/W/E/B/X classifications. They map subjects to native artifacts without claiming one Go test per TypeScript assertion or turning documentation counts into executed coverage. The same audit hash also binds reviewedGuides for formal and Go documentation; those guide reviews are freshness/scope records, not additional behavioral applicability rows. Per-case Quint and replay evidence is recorded separately.", "sources": [ { "path": "README.md", - "sha256": "17cd061cde252e466ba94fced8615d3bc7d2a6b88cced14fe62c3572dc86c173", + "sha256": "73fd3028635e25c046ff3af486f58ea18b2754038b60c58e12c7d9cc2624bfa5", "entries": 6, "contracts": [ "B01", diff --git a/formal/source-audit.json b/formal/source-audit.json index 2566c583..ba477d2e 100644 --- a/formal/source-audit.json +++ b/formal/source-audit.json @@ -3,7 +3,7 @@ "sources": [ { "path": "README.md", - "sha256": "17cd061cde252e466ba94fced8615d3bc7d2a6b88cced14fe62c3572dc86c173", + "sha256": "73fd3028635e25c046ff3af486f58ea18b2754038b60c58e12c7d9cc2624bfa5", "entries": [ {"line":1,"title":"DialCache","contracts":["B01"]}, {"line":24,"title":"Install","contracts":["B01"]}, diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 00000000..16fb33c3 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,167 @@ +# DialCache for Rust + +Rust implements the same portable behavior as the TypeScript library and the +Go port: explicit request enablement, request/local/Redis layers, +deterministic rollout, sparse runtime policy, request and process coalescing, +tracked invalidation, source and read deadlines, stale recovery, dark and +served-hit shadow validation, compression, and failure-isolated observability. + +The [Quint models](../formal/README.md) are the behavioral source of truth. +The Rust conformance harness replays the same sampled histories and named +public-action regressions as the other ports, plus the fixed scenarios and +Quint-derived protocol vectors, through the shared Node replay coordinator. +These are finite checks of the documented contract, not proof of every +possible input or schedule. + +## Use + +The crate requires Rust 1.85 or later; CI pins 1.98.1 through +`rust-toolchain.toml`. Applications own their Redis connection and its +timeout, retry and resource budgets. The default runtime is tokio. + +```rust +use std::sync::Arc; +use dialcache::{DialCache, KeySpec, Policy}; + +let cache = DialCache::builder() + .remote(dialcache::redis::RedisAdapter::new(connection)) // feature "redis" + .build()?; + +let display_name = cache + .use_case::("user", "displayName") + .policy(Policy::default().local_ttl_sec(1).remote_ttl_sec(60)) + .tracked(true) + .key(|id: &u64| KeySpec::new(id)) + .source(|_scope, id: u64| async move { Ok(load_display_name(id).await?) }) + .register()?; + +let request = cache.enable_guard(); +let name: Arc = display_name.get(request.scope(), 42).await?; +``` + +Caching is disabled by default. `DialCache::enable` (closure form) or +`DialCache::enable_guard` (RAII form) opens the outermost enabled scope and +hands out a `Scope`; pass it to every cached call made on behalf of that +request, including calls made inside a source. Completing the callback or +dropping the guard closes the scope: retained `Scope` clones no longer enable +caching and late work cannot publish into the request memo. +`DialCache::enable_in` and `DialCache::disable_in` derive nested scopes that +share the outer request memo. `Scope::outside()` is the pass-through scope of +work that runs on behalf of no request. + +`use_case` registers a typed use case once per instance and returns a +`UseCase` handle; `get_or_load` runs one inline `Operation` without +registration. Both snapshot the static policy and the source budget before any +asynchronous work. Values come back as `Arc`: shared by reference, treat +them as immutable. Use case `watermark` is reserved. `coalescing_state` +reports actual process leaders, followers and the oldest leader age. + +Sources are `Fn(Scope, Args) -> Future>`. They may +run again later for served-hit shadow validation, so they must be reusable. +Source errors surface as `Error::Source(Arc)`; every coalesced +caller receives the same shared instance, so `Arc::ptr_eq` identifies one +failure. A source deadline returns `Error::FallbackTimeout` and does not cancel +the source. Dropping the future returned by `get` never cancels the execution: +sources, publications and other callers keep their contracts. + +`KeySpec` takes the entity id and ordered secondary dimensions; +`normalize_args` spells scalars the JavaScript way and orders names by UTF-16 +code units so the same identity produces the same Redis key in every language. +Use the same namespace, key dimensions, codecs and policy across languages when +sharing entries. + +## Configuration and effects + +`Policy` holds the static leaves: whole-second TTLs (`local_ttl_sec`, +`remote_ttl_sec`), serving ramps, `request_local`, `coalesce`, +`stale_on_error_max_age_sec`, `remote_read_timeout_ms` and `shadow`. +`Policy::from_json` accepts the TypeScript JSON-shaped configuration. +`Policy::enabled(ttl)` and `Policy::disabled()` are the two static helpers. +A `policy_provider` returns a sparse `RuntimePolicy` overlay once per enabled +invocation; `Ok(None)` inherits, present leaves replace operation leaves, and +invalid leaves have the narrower consequences defined in Quint (an invalid TTL +or ramp disables only that layer; an invalid flag or read deadline bypasses +caching for the call). + +Defaults are namespace `urn`, local capacity 10,000, 50 ms remote reads, +60,000 ms source calls (`SourceBudget::Default`; `SourceBudget::Unbounded` +disables the deadline), sharing enabled, and shadow capacity one. Policy omits +all cache layers by default. Invalid constructor configuration returns +`ConfigError` from `build`; invalid operation configuration is returned before +execution. + +`Remote` supplies atomic primary snapshots, complete client-stamped frame +writes and surfaced invalidation errors. Writes are one native `SET`; +invalidation dispatches `EVALSHA` and retries once with `EVAL`. No value write +creates or extends a watermark. `DialCache::invalidate` affects shared remote +authority; other processes' local entries and already acquired snapshots +retain the documented lifetime rules. + +`Clock` separates wall time from elapsed time; `Runtime` supplies detached +task admission and timers. The defaults are `SystemClock` (aligned to the +process-wide millisecond grid used by local expiry) and `TokioRuntime`. The +`test-util` feature ships `testing::TestExecutor`, a deterministic +single-threaded executor with a virtual clock that runs the cache's detached +work to quiescence on demand and delivers timers only when a test advances +time; the conformance harness is built on it. + +## Values, codecs and observability + +`JsonCodec` (serde_json) is the default; `Codec` is asynchronous, and +`FromSync` adapts a synchronous `SyncCodec`. The TypeScript `undefined` +sentinel decodes as JSON `null`, so `Option` destinations read it as +`None`. Compression defaults to a 4,096-byte threshold and zstd level 3; +`disable_compression` stores payloads raw while reads still accept compressed +entries. The wire contract requires interoperable decompression, not identical +compressed bytes. + +`Observer` receives every public diagnostic as a typed `Event`. Shadow +validation exists only to be observed, so a job is admitted only when the +observer opts in through `observes_shadow_outcomes`; the bundled exporters do. +`Logger` receives structured `LogEvent`s and defaults to the `log` facade. +Mismatch logging is opt-in, confirmed, bounded, and previews values through +the operation's `preview` (JSON for serde values). Observer and logger failures +never change a cache, source or maintenance result. + +## Validation and reproducing a trace + +Use the repository [Make targets](../Makefile) from its root. CI pins Rust +1.98.1, Go 1.27.1, Node 24 and pnpm 10.33.0. Rust conformance tests use the +shared Node replay coordinator for command mappings and assertions; the crate +itself has no Node dependency. + +```sh +make check-rust # fmt, clippy, unit tests, protocol vectors, fixed scenarios and committed smoke histories +make formal # Quint model checks, full corpus, then TypeScript, Go and Rust replay +make formal-rust # Complete prepared Rust replay of the generated corpus +``` + +Without overrides, `cargo test --all-features --test conformance` replays the +committed smoke histories, every fixed scenario and every protocol vector. +The same `DIALCACHE_*_TRACE_DIR` / `_TRACE_FILE` selectors as Go replay a +directory or one history; `DIALCACHE_WITNESS_EVIDENCE_DIR` binds the shared +witness evidence and `DIALCACHE_RUST_REPORT` names the JSONL assertion report +the completion checker consumes. Reports and traces are kept in +`.formal-traces/`. + +```sh +DIALCACHE_FEATURE_TRACE_FILE="$PWD/.formal-traces/regressions/shadow/confirmationPastFreshnessKeepsOriginalPayloadAndAgeTest.itf.json" \ + cargo test --manifest-path rust/Cargo.toml --all-features --test conformance +``` + +`tests/settlement_control.rs` is the no-settle control required of every +port: a driver that reports observations before the settlement drain fails +every behavior-driver-backed smoke history. + +## Adaptations + +- Explicit `Scope` handles replace the implicit async context of TypeScript + and the `context.Context` of Go. +- Values are `Arc`; sources return `Result`. +- The default shadow comparator is `PartialEq`; `NaN` therefore differs from + itself where the TypeScript default treats it as equal. +- Local storage failures are exposed through the `LocalStore` trait rather + than a clock fault. +- The core replay, the Prometheus exporter and the Redis adapter follow their + Go counterparts; native Redis integration is not yet part of the completion + claim (see `formal/profiles.json`). From 6c6d2ea4e1c716f4ba4b57e627b62d13c688ac7a Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 01:04:45 -0700 Subject: [PATCH 13/31] docs: describe the Rust port in AGENTS.md --- AGENTS.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 5ca0da6b..6e29d6e7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Project overview -DialCache has TypeScript and Go implementations with explicit request-scoped enablement, local and Redis layers, runtime rollout controls, request coalescing, targeted invalidation, and adapter-based observability. +DialCache has TypeScript, Go and Rust implementations with explicit request-scoped enablement, local and Redis layers, runtime rollout controls, request coalescing, targeted invalidation, and adapter-based observability. ## Structure @@ -27,6 +27,7 @@ src/ internal/ # Cache layers, runtime config, payload compression, and invalidation Lua script test/ # Unit and Redis integration tests go/ # Go module, public cache and adapters, shared-corpus replay +rust/ # Rust crate, public cache and adapters, shared-corpus replay (tests/conformance.rs) formal/ # Quint behavioral source of truth, contracts and portable vectors ``` @@ -64,8 +65,8 @@ formal/ # Quint behavioral source of truth, contracts and portab readable as behavior definitions, share helpers with identical meaning, retain independent property checks, and register executable evidence in the catalogs. - Define portable behavior in Quint first. Require consequential generated - witnesses and replay the same histories in TypeScript and Go; keep native - API, wire and integration tests for their explicit boundaries. + witnesses and replay the same histories in TypeScript, Go and Rust; keep + native API, wire and integration tests for their explicit boundaries. ## Validation @@ -75,8 +76,10 @@ make check make integration ``` -Use `make formal` for complete Quint model checks, corpus generation and both -ports' full replay, then `make mutations` for assertion-strength checks. +`make check-rust` runs the Rust crate's fmt, clippy, unit, vector, scenario and +smoke checks; `make formal-rust` completes its replay of the generated corpus. +Use `make formal` for complete Quint model checks, corpus generation and every +port's full replay, then `make mutations` for assertion-strength checks. `make ci` runs all validation in the required order. `make help` lists targets and prerequisites; `formal/README.md` documents the fast PR and full-validation workflows. Full behavior/model/replay changes require full validation before From 4a408a6e86f72e40bd1d95ec1862f21c071c7696 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 01:10:51 -0700 Subject: [PATCH 14/31] feat(rust): Prometheus and Datadog metric exporters with the shared event-to-metric mapping Add metrics.rs (MetricKind: the nineteen kinds, label derivation in wire order, values in seconds/bytes/ratio), prometheus.rs (feature "prometheus": the nineteen collectors with the TypeScript names, help, labels and buckets; reuse per registry and prefix; conflicts error and roll back) and datadog.rs (DogStatsD client trait, namespace validation, 200-character name limit, histogram or distribution observations). Both observers opt in to shadow outcomes. Tests port the Go names/units/labels, wire-schema, reuse and conflict-isolation checks and prove a panicking observer never changes a cached result. --- rust/src/datadog.rs | 237 ++++++++++ rust/src/lib.rs | 10 + rust/src/metrics.rs | 602 ++++++++++++++++++++++++ rust/src/prometheus.rs | 440 ++++++++++++++++++ rust/tests/metrics_exporters.rs | 801 ++++++++++++++++++++++++++++++++ 5 files changed, 2090 insertions(+) create mode 100644 rust/src/datadog.rs create mode 100644 rust/src/metrics.rs create mode 100644 rust/src/prometheus.rs create mode 100644 rust/tests/metrics_exporters.rs diff --git a/rust/src/datadog.rs b/rust/src/datadog.rs new file mode 100644 index 00000000..ae86dd38 --- /dev/null +++ b/rust/src/datadog.rs @@ -0,0 +1,237 @@ +//! Datadog (DogStatsD) metric exporter. +//! +//! [`DatadogObserver`] publishes every [`Event`] to a caller-supplied +//! [`DogStatsdClient`] under the metric names, units and tags of the +//! TypeScript and Go adapters. Transport, buffering, flushing and ownership of +//! the client stay with the caller. + +use std::fmt; + +use crate::metrics::MetricKind; +use crate::observe::{Event, Observer}; + +/// The subset of the DogStatsD client API the exporter uses. +/// +/// `tags` are `(name, value)` pairs in wire order. Implementations format +/// them as `name:value` for their transport. +pub trait DogStatsdClient: Send + Sync + 'static { + /// Add `value` to the counter `name`. + fn increment(&self, name: &str, value: f64, tags: &[(String, String)]); + /// Record `value` in the histogram `name`. + fn histogram(&self, name: &str, value: f64, tags: &[(String, String)]); + /// Record `value` in the distribution `name`. + fn distribution(&self, name: &str, value: f64, tags: &[(String, String)]); +} + +/// How observations (timers, ages, sizes and ratios) are published. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ObservationMetricType { + /// Agent-side percentiles per host. + Histogram, + /// Server-side global percentiles. + Distribution, +} + +/// Construction options of a [`DatadogObserver`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DatadogOptions { + pub observation_metric_type: ObservationMetricType, + /// Metric-name namespace; unrelated to the cache namespace label. + /// `None` selects [`DEFAULT_NAMESPACE`]; an explicit empty string is an error. + pub namespace: Option, +} + +impl DatadogOptions { + pub fn new(observation_metric_type: ObservationMetricType) -> Self { + DatadogOptions { + observation_metric_type, + namespace: None, + } + } + + pub fn namespace(mut self, namespace: impl Into) -> Self { + self.namespace = Some(namespace.into()); + self + } +} + +/// The namespace used when [`DatadogOptions::namespace`] is `None`. +pub const DEFAULT_NAMESPACE: &str = "dialcache"; +/// Datadog's metric name length limit. +pub const METRIC_NAME_MAX_LENGTH: usize = 200; + +/// Construction failures of a [`DatadogObserver`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DatadogError { + /// The namespace does not match `^[A-Za-z][A-Za-z0-9_]*(\.[A-Za-z0-9_]+)*$`. + InvalidNamespace(String), + /// A metric name would exceed [`METRIC_NAME_MAX_LENGTH`] characters. + MetricNameTooLong(String), +} + +impl fmt::Display for DatadogError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + DatadogError::InvalidNamespace(namespace) => write!( + f, + "Datadog namespace {namespace:?} must start with a letter and contain only \ + letters, numbers, underscores, and dot-separated non-empty segments" + ), + DatadogError::MetricNameTooLong(name) => write!( + f, + "Datadog metric name {name:?} exceeds the {METRIC_NAME_MAX_LENGTH}-character limit" + ), + } + } +} + +impl std::error::Error for DatadogError {} + +/// The metric-name suffix of each kind, appended to the namespace. +pub fn metric_suffix(kind: MetricKind) -> &'static str { + match kind { + MetricKind::Request => "request.count", + MetricKind::Miss => "miss.count", + MetricKind::Disabled => "disabled.count", + MetricKind::Error => "error.count", + MetricKind::Invalidation => "invalidation.count", + MetricKind::Coalesced => "coalesced.count", + MetricKind::ShadowValidation => "shadow.count", + MetricKind::ShadowValueAge => "shadow.value_age", + MetricKind::FutureTimestampOffset => "future_timestamp_offset", + MetricKind::StaleRecovery => "stale_recovery.count", + MetricKind::StaleRecoveryValueAge => "stale_recovery.value_age", + MetricKind::Compression => "compression.count", + MetricKind::Get => "get.duration", + MetricKind::Fallback => "fallback.duration", + MetricKind::Serialization => "serialization.duration", + MetricKind::Size => "serialization.size", + MetricKind::StoredSize => "stored.size", + MetricKind::CompressionRatio => "compression.ratio", + MetricKind::CompressionDuration => "compression.duration", + } +} + +/// Whether `namespace` matches `^[A-Za-z][A-Za-z0-9_]*(\.[A-Za-z0-9_]+)*$`. +pub fn is_valid_namespace(namespace: &str) -> bool { + let mut segments = namespace.split('.'); + let Some(first) = segments.next() else { + return false; + }; + if !first + .chars() + .next() + .is_some_and(|c| c.is_ascii_alphabetic()) + { + return false; + } + let segment_ok = |segment: &str| { + !segment.is_empty() + && segment + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_') + }; + segment_ok(first) && segments.all(segment_ok) +} + +/// Publishes DialCache diagnostics to DogStatsD. Counters increment by one +/// per event; observations use the configured metric type. +pub struct DatadogObserver { + client: Box, + names: Vec, + distribution: bool, +} + +impl fmt::Debug for DatadogObserver { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("DatadogObserver") + .field("distribution", &self.distribution) + .field("names", &self.names) + .finish_non_exhaustive() + } +} + +impl DatadogObserver { + pub fn new( + client: impl DogStatsdClient, + options: DatadogOptions, + ) -> Result { + let namespace = options.namespace.as_deref().unwrap_or(DEFAULT_NAMESPACE); + if !is_valid_namespace(namespace) { + return Err(DatadogError::InvalidNamespace(namespace.to_string())); + } + let mut names = Vec::with_capacity(MetricKind::ALL.len()); + for kind in MetricKind::ALL { + let name = format!("{namespace}.{}", metric_suffix(kind)); + if name.chars().count() > METRIC_NAME_MAX_LENGTH { + return Err(DatadogError::MetricNameTooLong(name)); + } + names.push(name); + } + Ok(DatadogObserver { + client: Box::new(client), + names, + distribution: options.observation_metric_type == ObservationMetricType::Distribution, + }) + } + + /// The full metric name of `kind` under this observer's namespace. + pub fn metric_name(&self, kind: MetricKind) -> &str { + &self.names[kind.index()] + } +} + +impl Observer for DatadogObserver { + fn observe(&self, event: &Event) { + let kind = MetricKind::of(event); + let name = self.metric_name(kind); + let tags: Vec<(String, String)> = MetricKind::labels(event) + .into_iter() + .map(|(label, value)| (label.to_string(), value)) + .collect(); + if kind.is_counter() { + self.client.increment(name, 1.0, &tags); + } else if self.distribution { + self.client + .distribution(name, MetricKind::value(event), &tags); + } else { + self.client.histogram(name, MetricKind::value(event), &tags); + } + } + + fn observes_shadow_outcomes(&self) -> bool { + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn namespace_pattern_matches_the_reference_regex() { + for valid in ["dialcache", "app.cache", "A1_.b_2", "a", "x.y.z"] { + assert!(is_valid_namespace(valid), "{valid}"); + } + for invalid in [ + "", + "1bad", + "has-dash", + "two..dots", + ".lead", + "trail.", + "_under", + "sp ace", + "ünï", + ] { + assert!(!is_valid_namespace(invalid), "{invalid}"); + } + } + + #[test] + fn every_kind_has_a_distinct_suffix() { + let suffixes: std::collections::HashSet<&str> = + MetricKind::ALL.iter().map(|k| metric_suffix(*k)).collect(); + assert_eq!(suffixes.len(), MetricKind::ALL.len()); + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 154a36ad..38ead772 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -12,6 +12,7 @@ pub mod cancel; pub mod clock; pub mod codec; +pub mod datadog; mod deadline; mod engine; pub mod error; @@ -20,10 +21,13 @@ mod flight; pub mod identity; pub mod limits; pub mod local; +pub mod metrics; pub mod observe; pub mod operation; pub mod policy; pub mod preview; +#[cfg(feature = "prometheus")] +pub mod prometheus; pub mod protocol; pub mod remote; pub mod runtime; @@ -33,9 +37,14 @@ mod shadow; pub mod testing; mod use_case; +#[cfg(feature = "prometheus")] +pub use self::prometheus::{CollectorSchema, PrometheusError, PrometheusObserver}; pub use cancel::CancelToken; pub use clock::{Clock, SystemClock}; pub use codec::{Codec, FromSync, JsonCodec, Payload, SyncCodec}; +pub use datadog::{ + DatadogError, DatadogObserver, DatadogOptions, DogStatsdClient, ObservationMetricType, +}; pub use engine::{ CoalescingState, DialCache, DialCacheBuilder, PolicyProvider, ProcessCoalescingState, ScopeGuard, @@ -43,6 +52,7 @@ pub use engine::{ pub use error::{BoxError, ConfigError, Error, FallbackTimeout, RemoteReadTimeout, SharedError}; pub use identity::{normalize_args, ArgValue, Identity, IdentityError, Keys}; pub use local::{LocalEntry, LocalStore, LruLocalStore, StoredValue}; +pub use metrics::MetricKind; pub use observe::{Event, Labels, LogEvent, LogLevel, Logger, Observer, ShadowMismatchDetails}; pub use operation::{Comparator, Operation, Preview, RecoveryPredicate, SourceBudget}; pub use policy::{ diff --git a/rust/src/metrics.rs b/rust/src/metrics.rs new file mode 100644 index 00000000..cf639148 --- /dev/null +++ b/rust/src/metrics.rs @@ -0,0 +1,602 @@ +//! The shared event-to-metric mapping of the bundled exporters. +//! +//! Every [`Event`] maps to exactly one [`MetricKind`]: a counter or an +//! observation (timer, age, size, ratio). The kind fixes the metric's label +//! set and value so the Prometheus and Datadog exporters, and the TypeScript +//! and Go ports, publish the same wire contract. Logical cache keys never +//! appear here: labels are bounded enumerations plus the namespace, use case +//! and key type. + +use crate::observe::Event; + +/// The metric an [`Event`] feeds. Variant names match the TypeScript adapter +/// method names and the Go `metricKinds` table. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MetricKind { + Request, + Miss, + Disabled, + Error, + Invalidation, + Coalesced, + ShadowValidation, + ShadowValueAge, + FutureTimestampOffset, + StaleRecovery, + StaleRecoveryValueAge, + Compression, + Get, + Fallback, + Serialization, + Size, + StoredSize, + CompressionRatio, + CompressionDuration, +} + +/// The base label names every layer-scoped metric starts with. +const LABEL_CACHE_NAMESPACE: &str = "cache_namespace"; +const LABEL_USE_CASE: &str = "use_case"; +const LABEL_KEY_TYPE: &str = "key_type"; + +impl MetricKind { + /// Every kind, in declaration order. + pub const ALL: [MetricKind; 19] = [ + MetricKind::Request, + MetricKind::Miss, + MetricKind::Disabled, + MetricKind::Error, + MetricKind::Invalidation, + MetricKind::Coalesced, + MetricKind::ShadowValidation, + MetricKind::ShadowValueAge, + MetricKind::FutureTimestampOffset, + MetricKind::StaleRecovery, + MetricKind::StaleRecoveryValueAge, + MetricKind::Compression, + MetricKind::Get, + MetricKind::Fallback, + MetricKind::Serialization, + MetricKind::Size, + MetricKind::StoredSize, + MetricKind::CompressionRatio, + MetricKind::CompressionDuration, + ]; + + /// The metric an event feeds. + pub fn of(event: &Event) -> MetricKind { + match event { + Event::Request { .. } => MetricKind::Request, + Event::Miss { .. } => MetricKind::Miss, + Event::Disabled { .. } => MetricKind::Disabled, + Event::Error { .. } => MetricKind::Error, + Event::Invalidation { .. } => MetricKind::Invalidation, + Event::Coalesced { .. } => MetricKind::Coalesced, + Event::ShadowValidation { .. } => MetricKind::ShadowValidation, + Event::ShadowValueAge { .. } => MetricKind::ShadowValueAge, + Event::FutureTimestampOffset { .. } => MetricKind::FutureTimestampOffset, + Event::StaleRecovery { .. } => MetricKind::StaleRecovery, + Event::StaleRecoveryValueAge { .. } => MetricKind::StaleRecoveryValueAge, + Event::Compression { .. } => MetricKind::Compression, + Event::Get { .. } => MetricKind::Get, + Event::Fallback { .. } => MetricKind::Fallback, + Event::Serialization { .. } => MetricKind::Serialization, + Event::Size { .. } => MetricKind::Size, + Event::StoredSize { .. } => MetricKind::StoredSize, + Event::CompressionRatio { .. } => MetricKind::CompressionRatio, + Event::CompressionDuration { .. } => MetricKind::CompressionDuration, + } + } + + /// The kind's name in the TypeScript adapter and the Go `metricKinds` table. + pub fn as_str(self) -> &'static str { + match self { + MetricKind::Request => "request", + MetricKind::Miss => "miss", + MetricKind::Disabled => "disabled", + MetricKind::Error => "error", + MetricKind::Invalidation => "invalidation", + MetricKind::Coalesced => "coalesced", + MetricKind::ShadowValidation => "shadowValidation", + MetricKind::ShadowValueAge => "shadowValueAge", + MetricKind::FutureTimestampOffset => "futureTimestampOffset", + MetricKind::StaleRecovery => "staleRecovery", + MetricKind::StaleRecoveryValueAge => "staleRecoveryValueAge", + MetricKind::Compression => "compression", + MetricKind::Get => "get", + MetricKind::Fallback => "fallback", + MetricKind::Serialization => "serialization", + MetricKind::Size => "size", + MetricKind::StoredSize => "storedSize", + MetricKind::CompressionRatio => "compressionRatio", + MetricKind::CompressionDuration => "compressionDuration", + } + } + + /// Counters increment by one per event; every other kind records + /// [`value`](Self::value). + pub fn is_counter(self) -> bool { + matches!( + self, + MetricKind::Request + | MetricKind::Miss + | MetricKind::Disabled + | MetricKind::Error + | MetricKind::Invalidation + | MetricKind::Coalesced + | MetricKind::ShadowValidation + | MetricKind::StaleRecovery + | MetricKind::Compression + ) + } + + /// Position of the kind in [`ALL`](Self::ALL). + pub fn index(self) -> usize { + self as usize + } + + /// The label names of this kind, in wire order: the Prometheus label + /// schema and the Datadog tag set of the TypeScript adapters. + pub fn label_names(self) -> &'static [&'static str] { + const BASE: &[&str] = &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "layer", + ]; + const OUTCOME: &[&str] = &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "outcome", + ]; + match self { + MetricKind::Request + | MetricKind::FutureTimestampOffset + | MetricKind::Get + | MetricKind::Fallback + | MetricKind::Size + | MetricKind::StoredSize + | MetricKind::CompressionRatio => BASE, + MetricKind::Miss | MetricKind::Disabled => &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "layer", + "reason", + ], + MetricKind::Error => &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "layer", + "error", + "in_fallback", + ], + MetricKind::Invalidation => &[LABEL_CACHE_NAMESPACE, LABEL_KEY_TYPE, "layer"], + MetricKind::Coalesced => &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "scope", + ], + MetricKind::ShadowValidation + | MetricKind::ShadowValueAge + | MetricKind::StaleRecovery + | MetricKind::StaleRecoveryValueAge => OUTCOME, + MetricKind::Compression => &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "layer", + "outcome", + ], + MetricKind::Serialization | MetricKind::CompressionDuration => &[ + LABEL_CACHE_NAMESPACE, + LABEL_USE_CASE, + LABEL_KEY_TYPE, + "layer", + "operation", + ], + } + } + + /// The labels of an event, in the order of [`label_names`](Self::label_names) + /// for [`of(event)`](Self::of). Values are bounded enumerations plus the + /// namespace, use case and key type; the logical key is never one. + pub fn labels(event: &Event) -> Vec<(&'static str, String)> { + let mut labels: Vec<(&'static str, String)> = Vec::with_capacity(6); + match event { + Event::Invalidation { + namespace, + key_type, + layer, + } => { + labels.push((LABEL_CACHE_NAMESPACE, namespace.to_string())); + labels.push((LABEL_KEY_TYPE, key_type.to_string())); + labels.push(("layer", layer.as_str().to_string())); + } + Event::Coalesced { + labels: outcome, + scope, + } => { + push_outcome_base(&mut labels, outcome); + labels.push(("scope", scope.as_str().to_string())); + } + Event::ShadowValidation { + labels: outcome, + outcome: verdict, + } + | Event::ShadowValueAge { + labels: outcome, + outcome: verdict, + .. + } => { + push_outcome_base(&mut labels, outcome); + labels.push(("outcome", verdict.as_str().to_string())); + } + Event::StaleRecovery { + labels: outcome, + outcome: verdict, + } + | Event::StaleRecoveryValueAge { + labels: outcome, + outcome: verdict, + .. + } => { + push_outcome_base(&mut labels, outcome); + labels.push(("outcome", verdict.as_str().to_string())); + } + Event::Request { labels: base } + | Event::FutureTimestampOffset { labels: base, .. } + | Event::Get { labels: base, .. } + | Event::Fallback { labels: base, .. } + | Event::Size { labels: base, .. } + | Event::StoredSize { labels: base, .. } + | Event::CompressionRatio { labels: base, .. } => push_layer_base(&mut labels, base), + Event::Miss { + labels: base, + reason, + } => { + push_layer_base(&mut labels, base); + labels.push(("reason", reason.as_str().to_string())); + } + Event::Disabled { + labels: base, + reason, + } => { + push_layer_base(&mut labels, base); + labels.push(("reason", reason.as_str().to_string())); + } + Event::Error { + labels: base, + error, + in_fallback, + } => { + push_layer_base(&mut labels, base); + labels.push(("error", error.as_str().to_string())); + labels.push(("in_fallback", in_fallback.to_string())); + } + Event::Compression { + labels: base, + outcome, + } => { + push_layer_base(&mut labels, base); + labels.push(("outcome", outcome.as_str().to_string())); + } + Event::Serialization { + labels: base, + operation, + .. + } => { + push_layer_base(&mut labels, base); + labels.push(("operation", operation.as_str().to_string())); + } + Event::CompressionDuration { + labels: base, + operation, + .. + } => { + push_layer_base(&mut labels, base); + labels.push(("operation", operation.as_str().to_string())); + } + } + labels + } + + /// The value an event records: seconds for timers, ages and offsets, + /// bytes for sizes, the compressed-to-original ratio for + /// [`CompressionRatio`](Self::CompressionRatio), and the increment `1` + /// for counters. + pub fn value(event: &Event) -> f64 { + match event { + Event::ShadowValueAge { seconds, .. } + | Event::FutureTimestampOffset { seconds, .. } + | Event::StaleRecoveryValueAge { seconds, .. } + | Event::Get { seconds, .. } + | Event::Fallback { seconds, .. } + | Event::Serialization { seconds, .. } + | Event::CompressionDuration { seconds, .. } => *seconds, + Event::Size { bytes, .. } | Event::StoredSize { bytes, .. } => *bytes as f64, + Event::CompressionRatio { ratio, .. } => *ratio, + Event::Request { .. } + | Event::Miss { .. } + | Event::Disabled { .. } + | Event::Error { .. } + | Event::Invalidation { .. } + | Event::Coalesced { .. } + | Event::ShadowValidation { .. } + | Event::StaleRecovery { .. } + | Event::Compression { .. } => 1.0, + } + } +} + +fn push_layer_base(labels: &mut Vec<(&'static str, String)>, base: &crate::observe::Labels) { + labels.push((LABEL_CACHE_NAMESPACE, base.namespace.to_string())); + labels.push((LABEL_USE_CASE, base.use_case.to_string())); + labels.push((LABEL_KEY_TYPE, base.key_type.to_string())); + labels.push(("layer", base.layer.as_str().to_string())); +} + +fn push_outcome_base( + labels: &mut Vec<(&'static str, String)>, + base: &crate::observe::OutcomeLabels, +) { + labels.push((LABEL_CACHE_NAMESPACE, base.namespace.to_string())); + labels.push((LABEL_USE_CASE, base.use_case.to_string())); + labels.push((LABEL_KEY_TYPE, base.key_type.to_string())); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::observe::{ + CoalescingScope, CompressionOperation, CompressionOutcome, DisabledReason, ErrorKind, + Labels, Layer, MissReason, OutcomeLabels, RecoveryOutcome, SerializationOperation, + ShadowOutcome, + }; + use std::sync::Arc; + + fn base() -> Labels { + Labels { + namespace: Arc::from("logical"), + use_case: Arc::from("lookup"), + key_type: Arc::from("item"), + layer: Layer::Remote, + } + } + + fn outcome() -> OutcomeLabels { + OutcomeLabels { + namespace: Arc::from("logical"), + use_case: Arc::from("lookup"), + key_type: Arc::from("item"), + } + } + + /// One event per kind with fixed, recognizable field values. + pub(crate) fn event_of(kind: MetricKind) -> Event { + match kind { + MetricKind::Request => Event::Request { labels: base() }, + MetricKind::Miss => Event::Miss { + labels: base(), + reason: MissReason::Expired, + }, + MetricKind::Disabled => Event::Disabled { + labels: base(), + reason: DisabledReason::RampedDown, + }, + MetricKind::Error => Event::Error { + labels: base(), + error: ErrorKind::Fallback, + in_fallback: true, + }, + MetricKind::Invalidation => Event::Invalidation { + namespace: Arc::from("logical"), + key_type: Arc::from("item"), + layer: Layer::Remote, + }, + MetricKind::Coalesced => Event::Coalesced { + labels: outcome(), + scope: CoalescingScope::Process, + }, + MetricKind::ShadowValidation => Event::ShadowValidation { + labels: outcome(), + outcome: ShadowOutcome::Mismatch, + }, + MetricKind::ShadowValueAge => Event::ShadowValueAge { + labels: outcome(), + outcome: ShadowOutcome::Mismatch, + seconds: 0.25, + }, + MetricKind::FutureTimestampOffset => Event::FutureTimestampOffset { + labels: base(), + seconds: 0.25, + }, + MetricKind::StaleRecovery => Event::StaleRecovery { + labels: outcome(), + outcome: RecoveryOutcome::Served, + }, + MetricKind::StaleRecoveryValueAge => Event::StaleRecoveryValueAge { + labels: outcome(), + outcome: RecoveryOutcome::Served, + seconds: 0.25, + }, + MetricKind::Compression => Event::Compression { + labels: base(), + outcome: CompressionOutcome::Compressed, + }, + MetricKind::Get => Event::Get { + labels: base(), + seconds: 0.25, + }, + MetricKind::Fallback => Event::Fallback { + labels: base(), + seconds: 0.25, + }, + MetricKind::Serialization => Event::Serialization { + labels: base(), + operation: SerializationOperation::Dump, + seconds: 0.25, + }, + MetricKind::Size => Event::Size { + labels: base(), + bytes: 123, + }, + MetricKind::StoredSize => Event::StoredSize { + labels: base(), + bytes: 123, + }, + MetricKind::CompressionRatio => Event::CompressionRatio { + labels: base(), + ratio: 0.25, + }, + MetricKind::CompressionDuration => Event::CompressionDuration { + labels: base(), + operation: CompressionOperation::Compress, + seconds: 0.25, + }, + } + } + + #[test] + fn every_kind_round_trips_through_of_and_index() { + for (index, kind) in MetricKind::ALL.iter().enumerate() { + assert_eq!(MetricKind::of(&event_of(*kind)), *kind); + assert_eq!(kind.index(), index); + } + let names: std::collections::HashSet<&str> = + MetricKind::ALL.iter().map(|k| k.as_str()).collect(); + assert_eq!(names.len(), 19); + } + + #[test] + fn label_values_follow_the_declared_label_names() { + for kind in MetricKind::ALL { + let labels = MetricKind::labels(&event_of(kind)); + let names: Vec<&str> = labels.iter().map(|(name, _)| *name).collect(); + assert_eq!(names, kind.label_names(), "{}", kind.as_str()); + } + } + + #[test] + fn labels_derive_as_the_go_port_does() { + let pairs = |kind: MetricKind| -> Vec<(String, String)> { + MetricKind::labels(&event_of(kind)) + .into_iter() + .map(|(k, v)| (k.to_string(), v)) + .collect() + }; + let owned = |items: &[(&str, &str)]| -> Vec<(String, String)> { + items + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() + }; + assert_eq!( + pairs(MetricKind::Error), + owned(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("layer", "remote"), + ("error", "fallback"), + ("in_fallback", "true"), + ]) + ); + assert_eq!( + pairs(MetricKind::Invalidation), + owned(&[ + ("cache_namespace", "logical"), + ("key_type", "item"), + ("layer", "remote"), + ]) + ); + assert_eq!( + pairs(MetricKind::Coalesced), + owned(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("scope", "process"), + ]) + ); + assert_eq!( + pairs(MetricKind::ShadowValidation), + owned(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("outcome", "mismatch"), + ]) + ); + assert_eq!( + pairs(MetricKind::Miss), + owned(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("layer", "remote"), + ("reason", "expired"), + ]) + ); + assert_eq!( + pairs(MetricKind::Disabled).last().unwrap(), + &("reason".to_string(), "ramped_down".to_string()) + ); + assert_eq!( + pairs(MetricKind::Compression).last().unwrap(), + &("outcome".to_string(), "compressed".to_string()) + ); + assert_eq!( + pairs(MetricKind::Serialization).last().unwrap(), + &("operation".to_string(), "dump".to_string()) + ); + assert_eq!( + pairs(MetricKind::CompressionDuration).last().unwrap(), + &("operation".to_string(), "compress".to_string()) + ); + let in_fallback_false = Event::Error { + labels: base(), + error: ErrorKind::CacheRead, + in_fallback: false, + }; + assert_eq!( + MetricKind::labels(&in_fallback_false).last().unwrap().1, + "false" + ); + } + + #[test] + fn values_carry_the_documented_units() { + for kind in MetricKind::ALL { + let value = MetricKind::value(&event_of(kind)); + let expected = match kind { + MetricKind::Size | MetricKind::StoredSize => 123.0, + _ if kind.is_counter() => 1.0, + _ => 0.25, + }; + assert_eq!(value, expected, "{}", kind.as_str()); + } + let counters: Vec<&str> = MetricKind::ALL + .iter() + .filter(|k| k.is_counter()) + .map(|k| k.as_str()) + .collect(); + assert_eq!( + counters, + [ + "request", + "miss", + "disabled", + "error", + "invalidation", + "coalesced", + "shadowValidation", + "staleRecovery", + "compression", + ] + ); + } +} diff --git a/rust/src/prometheus.rs b/rust/src/prometheus.rs new file mode 100644 index 00000000..17b4640d --- /dev/null +++ b/rust/src/prometheus.rs @@ -0,0 +1,440 @@ +//! Prometheus metric exporter (feature `prometheus`). +//! +//! [`PrometheusObserver`] registers the nineteen DialCache collectors on a +//! [`Registry`] under the names, help text, label sets and histogram buckets +//! of the TypeScript adapter, and feeds every [`Event`] to them. +//! +//! # Reuse +//! +//! Collectors are process-wide state: constructing a second observer for the +//! same registry and prefix reuses the collectors the first one created, so +//! observations from both accumulate in one series set. The `prometheus` +//! crate reports a duplicate registration without a handle to the existing +//! collector, so reuse is keyed by the registry's address and the prefix in a +//! process-wide table. Pass the same [`Registry`] value each time, not a +//! clone: a clone shares the registered collectors but has another address, +//! and its construction fails as an externally owned duplicate. +//! +//! An externally registered collector with one of the DialCache names is +//! reported as [`PrometheusError::Conflict`]; the constructor then leaves the +//! registry as it found it. + +use std::collections::HashMap; +use std::fmt; +use std::sync::{Arc, LazyLock}; + +use ::prometheus::core::Collector; +use ::prometheus::{HistogramOpts, HistogramVec, IntCounterVec, Opts, Registry}; +use parking_lot::Mutex; + +use crate::metrics::MetricKind; +use crate::observe::{Event, Observer}; + +const TIMER_BUCKETS: &[f64] = &[ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, +]; +const SIZE_BUCKETS: &[f64] = &[ + 100.0, + 1_000.0, + 10_000.0, + 100_000.0, + 1_000_000.0, + 10_000_000.0, +]; +const RATIO_BUCKETS: &[f64] = &[0.05, 0.1, 0.2, 0.3, 0.5, 0.7, 0.9, 1.0]; +/// Value ages span seconds to the 365-day TTL ceiling: 1s..15m, then 1h, 3h, 12h, 1d, 3d, 7d. +const VALUE_AGE_BUCKETS: &[f64] = &[ + 1.0, 5.0, 15.0, 60.0, 300.0, 900.0, 3_600.0, 10_800.0, 43_200.0, 86_400.0, 259_200.0, 604_800.0, +]; +const FUTURE_TIMESTAMP_OFFSET_BUCKETS: &[f64] = &[ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 15.0, 60.0, 300.0, 900.0, 3_600.0, + 10_800.0, 43_200.0, +]; + +/// The wire schema of one collector: what a scrape exposes. +#[derive(Debug, Clone, PartialEq)] +pub struct CollectorSchema { + pub kind: MetricKind, + /// The fully qualified metric name, prefix included. + pub name: String, + pub help: &'static str, + /// Label names in wire order. + pub labels: &'static [&'static str], + /// Histogram bucket upper bounds; empty for counters. + pub buckets: &'static [f64], +} + +impl CollectorSchema { + pub fn is_counter(&self) -> bool { + self.kind.is_counter() + } +} + +/// The nineteen collectors DialCache registers, in registration order, with +/// their names under `prefix`. +pub fn schemas(prefix: &str) -> Vec { + let schema = |kind: MetricKind, + suffix: &str, + help: &'static str, + buckets: &'static [f64]| + -> CollectorSchema { + CollectorSchema { + kind, + name: format!("{prefix}dialcache_{suffix}"), + help, + labels: kind.label_names(), + buckets, + } + }; + vec![ + schema( + MetricKind::Disabled, + "disabled_counter", + "Requests where DialCache skipped a cache layer.", + &[], + ), + schema( + MetricKind::Miss, + "miss_counter", + "DialCache cache misses.", + &[], + ), + schema( + MetricKind::Request, + "request_counter", + "Total DialCache cache-layer requests.", + &[], + ), + schema( + MetricKind::Error, + "error_counter", + "Errors during DialCache cache operations or fallback execution.", + &[], + ), + schema( + MetricKind::Invalidation, + "invalidation_counter", + "DialCache invalidation calls by key type and layer.", + &[], + ), + schema( + MetricKind::Coalesced, + "coalesced_counter", + "DialCache requests coalesced onto in-flight work by sharing scope.", + &[], + ), + schema( + MetricKind::ShadowValidation, + "shadow_validation_counter", + "Sampled DialCache Redis shadow-validation outcomes.", + &[], + ), + schema( + MetricKind::ShadowValueAge, + "shadow_value_age_histogram", + "Age in seconds of the validated Redis value at DialCache shadow verdict time.", + VALUE_AGE_BUCKETS, + ), + schema( + MetricKind::FutureTimestampOffset, + "future_timestamp_offset_histogram", + "Positive offset in seconds of Redis frames dated after the observing DialCache process clock.", + FUTURE_TIMESTAMP_OFFSET_BUCKETS, + ), + schema( + MetricKind::StaleRecovery, + "stale_recovery_counter", + "DialCache stale-on-error Redis recovery outcomes.", + &[], + ), + schema( + MetricKind::StaleRecoveryValueAge, + "stale_recovery_value_age_histogram", + "Age in seconds of Redis values served by DialCache stale-on-error recovery.", + VALUE_AGE_BUCKETS, + ), + schema( + MetricKind::Compression, + "compression_counter", + "DialCache Redis payload compression and decompression outcomes.", + &[], + ), + schema( + MetricKind::Get, + "get_timer", + "DialCache cache get latency in seconds.", + TIMER_BUCKETS, + ), + schema( + MetricKind::Fallback, + "fallback_timer", + "Time DialCache waited for the fallback function in seconds.", + TIMER_BUCKETS, + ), + schema( + MetricKind::Serialization, + "serialization_timer", + "DialCache serialization latency in seconds.", + TIMER_BUCKETS, + ), + schema( + MetricKind::Size, + "size_histogram", + "Serialized DialCache value sizes in bytes.", + SIZE_BUCKETS, + ), + schema( + MetricKind::StoredSize, + "stored_size_histogram", + "Stored DialCache payload sizes in bytes, after compression and escaping.", + SIZE_BUCKETS, + ), + schema( + MetricKind::CompressionRatio, + "compression_ratio_histogram", + "Compressed-to-original DialCache payload size ratio for compressed writes.", + RATIO_BUCKETS, + ), + schema( + MetricKind::CompressionDuration, + "compression_timer", + "DialCache payload compression and decompression latency in seconds.", + TIMER_BUCKETS, + ), + ] +} + +/// Construction failures of a [`PrometheusObserver`]. +#[derive(Debug)] +pub enum PrometheusError { + /// A collector with this name is already registered by someone else, or + /// with another schema. Use a unique prefix or another registry. + Conflict { + name: String, + source: ::prometheus::Error, + }, + /// The `prometheus` crate rejected a collector definition, which only a + /// prefix that is not a valid metric name fragment can cause. + InvalidCollector { + name: String, + source: ::prometheus::Error, + }, +} + +impl fmt::Display for PrometheusError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + PrometheusError::Conflict { name, source } => write!( + f, + "Prometheus collector {name:?} already exists with an incompatible or externally \ + owned schema; use a unique prefix or registry: {source}" + ), + PrometheusError::InvalidCollector { name, source } => { + write!(f, "invalid Prometheus collector {name:?}: {source}") + } + } + } +} + +impl std::error::Error for PrometheusError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + PrometheusError::Conflict { source, .. } + | PrometheusError::InvalidCollector { source, .. } => Some(source), + } + } +} + +#[derive(Clone)] +enum Vector { + Counter(IntCounterVec), + Histogram(HistogramVec), +} + +impl Vector { + fn boxed(&self) -> Box { + match self { + Vector::Counter(vector) => Box::new(vector.clone()), + Vector::Histogram(vector) => Box::new(vector.clone()), + } + } +} + +/// The collectors of one (registry, prefix) pair, indexed by [`MetricKind::index`]. +struct Group { + vectors: Vec, +} + +impl Group { + fn build(prefix: &str) -> Result { + let mut vectors: Vec> = vec![None; MetricKind::ALL.len()]; + for schema in schemas(prefix) { + let vector = if schema.is_counter() { + IntCounterVec::new(Opts::new(&schema.name, schema.help), schema.labels) + .map(Vector::Counter) + } else { + HistogramVec::new( + HistogramOpts::new(&schema.name, schema.help).buckets(schema.buckets.to_vec()), + schema.labels, + ) + .map(Vector::Histogram) + } + .map_err(|source| PrometheusError::InvalidCollector { + name: schema.name.clone(), + source, + })?; + vectors[schema.kind.index()] = Some(vector); + } + Ok(Group { + vectors: vectors + .into_iter() + .map(|vector| vector.expect("every kind has a schema")) + .collect(), + }) + } + + /// Register every collector on `registry`; roll back on the first failure. + fn register(&self, registry: &Registry, prefix: &str) -> Result<(), PrometheusError> { + let mut registered: Vec<&Vector> = Vec::new(); + for schema in schemas(prefix) { + let vector = &self.vectors[schema.kind.index()]; + if let Err(source) = registry.register(vector.boxed()) { + for done in registered { + let _ = registry.unregister(done.boxed()); + } + return Err(PrometheusError::Conflict { + name: schema.name, + source, + }); + } + registered.push(vector); + } + Ok(()) + } + + /// Make sure every collector is registered on `registry`, accepting the + /// ones already there. + fn ensure_registered(&self, registry: &Registry, prefix: &str) -> Result<(), PrometheusError> { + for schema in schemas(prefix) { + match registry.register(self.vectors[schema.kind.index()].boxed()) { + Ok(()) | Err(::prometheus::Error::AlreadyReg) => {} + Err(source) => { + return Err(PrometheusError::Conflict { + name: schema.name, + source, + }) + } + } + } + Ok(()) + } +} + +type GroupKey = (usize, String); + +/// Collector groups by registry address and prefix. Groups are retained for +/// the process lifetime so a later observer for the same registry finds the +/// collectors even after every earlier observer was dropped. +static GROUPS: LazyLock>>> = + LazyLock::new(|| Mutex::new(HashMap::new())); + +/// Publishes DialCache diagnostics to Prometheus collectors. +pub struct PrometheusObserver { + group: Arc, + prefix: String, +} + +impl fmt::Debug for PrometheusObserver { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PrometheusObserver") + .field("prefix", &self.prefix) + .finish_non_exhaustive() + } +} + +impl PrometheusObserver { + /// Create or reuse the DialCache collectors named `dialcache_*` + /// on `registry`. See the module documentation for the reuse rule. + pub fn new(registry: &Registry, prefix: &str) -> Result { + let key: GroupKey = (registry as *const Registry as usize, prefix.to_string()); + let mut groups = GROUPS.lock(); + let group = match groups.get(&key) { + Some(group) => { + group.ensure_registered(registry, prefix)?; + group.clone() + } + None => { + let group = Arc::new(Group::build(prefix)?); + group.register(registry, prefix)?; + groups.insert(key, group.clone()); + group + } + }; + Ok(PrometheusObserver { + group, + prefix: prefix.to_string(), + }) + } + + /// The wire schemas of this observer's collectors. + pub fn schemas(&self) -> Vec { + schemas(&self.prefix) + } +} + +impl Observer for PrometheusObserver { + fn observe(&self, event: &Event) { + let kind = MetricKind::of(event); + let labels = MetricKind::labels(event); + let values: Vec<&str> = labels.iter().map(|(_, value)| value.as_str()).collect(); + let result = match &self.group.vectors[kind.index()] { + Vector::Counter(vector) => vector + .get_metric_with_label_values(&values) + .map(|metric| metric.inc()), + Vector::Histogram(vector) => vector + .get_metric_with_label_values(&values) + .map(|metric| metric.observe(MetricKind::value(event))), + }; + if let Err(error) = result { + log::warn!( + target: "dialcache", + "Dropped DialCache {} metric with labels {:?}: {error}", + kind.as_str(), + labels + ); + } + } + + fn observes_shadow_outcomes(&self) -> bool { + true + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn schemas_cover_every_kind_once_with_matching_label_names() { + let all = schemas("p_"); + assert_eq!(all.len(), MetricKind::ALL.len()); + let mut seen = std::collections::HashSet::new(); + for schema in &all { + assert!(seen.insert(schema.kind), "{:?} twice", schema.kind); + assert!(schema.name.starts_with("p_dialcache_")); + assert_eq!(schema.labels, schema.kind.label_names()); + assert_eq!(schema.buckets.is_empty(), schema.is_counter()); + } + } + + #[test] + fn invalid_prefix_is_an_error_not_a_panic() { + let registry = Registry::new(); + let error = PrometheusObserver::new(®istry, "bad prefix ").unwrap_err(); + assert!( + matches!(error, PrometheusError::InvalidCollector { .. }), + "{error}" + ); + assert!(registry.gather().is_empty()); + } +} diff --git a/rust/tests/metrics_exporters.rs b/rust/tests/metrics_exporters.rs new file mode 100644 index 00000000..18038288 --- /dev/null +++ b/rust/tests/metrics_exporters.rs @@ -0,0 +1,801 @@ +//! The metric exporters' wire contract: Datadog names, units and tags; +//! Prometheus schemas, reuse and conflict isolation; and observer failure +//! isolation from cache results. + +use std::sync::Arc; + +use dialcache::datadog::{metric_suffix, DatadogError}; +use dialcache::observe::{ + CoalescingScope, CompressionOperation, CompressionOutcome, DisabledReason, ErrorKind, Labels, + Layer, MissReason, OutcomeLabels, RecoveryOutcome, SerializationOperation, ShadowOutcome, +}; +use dialcache::{ + DatadogObserver, DatadogOptions, DogStatsdClient, Event, MetricKind, ObservationMetricType, + Observer, +}; +use parking_lot::Mutex; + +const NEVER_EXPORTED: &str = "never_export_this_identity"; + +fn base() -> Labels { + Labels { + namespace: Arc::from("logical"), + use_case: Arc::from("lookup"), + key_type: Arc::from("item"), + layer: Layer::Remote, + } +} + +fn outcome() -> OutcomeLabels { + OutcomeLabels { + namespace: Arc::from("logical"), + use_case: Arc::from("lookup"), + key_type: Arc::from("item"), + } +} + +/// One event per kind, mirroring Go's `metricTestEvent`: 0.25 s timers and +/// ages, 123-byte sizes, a 0.25 ratio. +fn metric_test_event(kind: MetricKind) -> Event { + match kind { + MetricKind::Request => Event::Request { labels: base() }, + MetricKind::Miss => Event::Miss { + labels: base(), + reason: MissReason::Expired, + }, + MetricKind::Disabled => Event::Disabled { + labels: base(), + reason: DisabledReason::RampedDown, + }, + MetricKind::Error => Event::Error { + labels: base(), + error: ErrorKind::Fallback, + in_fallback: true, + }, + MetricKind::Invalidation => Event::Invalidation { + namespace: Arc::from("logical"), + key_type: Arc::from("item"), + layer: Layer::Remote, + }, + MetricKind::Coalesced => Event::Coalesced { + labels: outcome(), + scope: CoalescingScope::Process, + }, + MetricKind::ShadowValidation => Event::ShadowValidation { + labels: outcome(), + outcome: ShadowOutcome::Mismatch, + }, + MetricKind::ShadowValueAge => Event::ShadowValueAge { + labels: outcome(), + outcome: ShadowOutcome::Mismatch, + seconds: 0.25, + }, + MetricKind::FutureTimestampOffset => Event::FutureTimestampOffset { + labels: base(), + seconds: 0.25, + }, + MetricKind::StaleRecovery => Event::StaleRecovery { + labels: outcome(), + outcome: RecoveryOutcome::Served, + }, + MetricKind::StaleRecoveryValueAge => Event::StaleRecoveryValueAge { + labels: outcome(), + outcome: RecoveryOutcome::Served, + seconds: 0.25, + }, + MetricKind::Compression => Event::Compression { + labels: base(), + outcome: CompressionOutcome::Compressed, + }, + MetricKind::Get => Event::Get { + labels: base(), + seconds: 0.25, + }, + MetricKind::Fallback => Event::Fallback { + labels: base(), + seconds: 0.25, + }, + MetricKind::Serialization => Event::Serialization { + labels: base(), + operation: SerializationOperation::Dump, + seconds: 0.25, + }, + MetricKind::Size => Event::Size { + labels: base(), + bytes: 123, + }, + MetricKind::StoredSize => Event::StoredSize { + labels: base(), + bytes: 123, + }, + MetricKind::CompressionRatio => Event::CompressionRatio { + labels: base(), + ratio: 0.25, + }, + MetricKind::CompressionDuration => Event::CompressionDuration { + labels: base(), + operation: CompressionOperation::Compress, + seconds: 0.25, + }, + } +} + +fn tags(items: &[(&str, &str)]) -> Vec<(String, String)> { + items + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect() +} + +#[derive(Debug, Clone, PartialEq)] +struct Recorded { + method: &'static str, + name: String, + value: f64, + tags: Vec<(String, String)>, +} + +#[derive(Default)] +struct RecordingClient { + events: Arc>>, +} + +impl RecordingClient { + fn record(&self, method: &'static str, name: &str, value: f64, tags: &[(String, String)]) { + self.events.lock().push(Recorded { + method, + name: name.to_string(), + value, + tags: tags.to_vec(), + }); + } +} + +impl DogStatsdClient for RecordingClient { + fn increment(&self, name: &str, value: f64, tags: &[(String, String)]) { + self.record("increment", name, value, tags); + } + fn histogram(&self, name: &str, value: f64, tags: &[(String, String)]) { + self.record("histogram", name, value, tags); + } + fn distribution(&self, name: &str, value: f64, tags: &[(String, String)]) { + self.record("distribution", name, value, tags); + } +} + +fn recording() -> (RecordingClient, Arc>>) { + let client = RecordingClient::default(); + let events = client.events.clone(); + (client, events) +} + +/// Port of TestDatadogMetricNamesUnitsAndLabels. +#[test] +fn datadog_metric_names_units_and_labels() { + let (client, events) = recording(); + let observer = DatadogObserver::new( + client, + DatadogOptions::new(ObservationMetricType::Distribution).namespace("app.cache"), + ) + .expect("valid options"); + assert!(observer.observes_shadow_outcomes()); + + // (kind, suffix, method, value), copied from go/metrics_test.go. + let cases: [(MetricKind, &str, &str, f64); 19] = [ + (MetricKind::Request, "request.count", "increment", 1.0), + (MetricKind::Miss, "miss.count", "increment", 1.0), + (MetricKind::Disabled, "disabled.count", "increment", 1.0), + (MetricKind::Error, "error.count", "increment", 1.0), + ( + MetricKind::Invalidation, + "invalidation.count", + "increment", + 1.0, + ), + (MetricKind::Coalesced, "coalesced.count", "increment", 1.0), + ( + MetricKind::ShadowValidation, + "shadow.count", + "increment", + 1.0, + ), + ( + MetricKind::ShadowValueAge, + "shadow.value_age", + "distribution", + 0.25, + ), + ( + MetricKind::FutureTimestampOffset, + "future_timestamp_offset", + "distribution", + 0.25, + ), + ( + MetricKind::StaleRecovery, + "stale_recovery.count", + "increment", + 1.0, + ), + ( + MetricKind::StaleRecoveryValueAge, + "stale_recovery.value_age", + "distribution", + 0.25, + ), + ( + MetricKind::Compression, + "compression.count", + "increment", + 1.0, + ), + (MetricKind::Get, "get.duration", "distribution", 0.25), + ( + MetricKind::Fallback, + "fallback.duration", + "distribution", + 0.25, + ), + ( + MetricKind::Serialization, + "serialization.duration", + "distribution", + 0.25, + ), + ( + MetricKind::Size, + "serialization.size", + "distribution", + 123.0, + ), + (MetricKind::StoredSize, "stored.size", "distribution", 123.0), + ( + MetricKind::CompressionRatio, + "compression.ratio", + "distribution", + 0.25, + ), + ( + MetricKind::CompressionDuration, + "compression.duration", + "distribution", + 0.25, + ), + ]; + for (kind, suffix, method, value) in cases { + observer.observe(&metric_test_event(kind)); + let got = events.lock().last().cloned().expect("recorded"); + assert_eq!(got.name, format!("app.cache.{suffix}"), "{kind:?}"); + assert_eq!(got.method, method, "{kind:?}"); + assert_eq!(got.value, value, "{kind:?}"); + assert_eq!(metric_suffix(kind), suffix); + assert_eq!(observer.metric_name(kind), got.name); + for (_, tag) in &got.tags { + assert!(!tag.contains("never_export"), "key leaked to metric labels"); + } + } + let recorded = events.lock().clone(); + assert_eq!( + recorded[3].tags, + tags(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("layer", "remote"), + ("error", "fallback"), + ("in_fallback", "true"), + ]), + "error tags" + ); + assert_eq!( + recorded[4].tags, + tags(&[ + ("cache_namespace", "logical"), + ("key_type", "item"), + ("layer", "remote"), + ]), + "invalidation acquired use_case" + ); + assert_eq!( + recorded[5].tags, + tags(&[ + ("cache_namespace", "logical"), + ("use_case", "lookup"), + ("key_type", "item"), + ("scope", "process"), + ]), + "coalescing labels changed" + ); + assert!( + !recorded[6].tags.iter().any(|(name, _)| name == "layer"), + "shadow acquired layer" + ); + assert_eq!( + recorded[6] + .tags + .last() + .map(|(k, v)| (k.as_str(), v.as_str())), + Some(("outcome", "mismatch")) + ); + + let (client, events) = recording(); + let histogram = DatadogObserver::new( + client, + DatadogOptions::new(ObservationMetricType::Histogram), + ) + .expect("default namespace"); + histogram.observe(&metric_test_event(MetricKind::Get)); + let got = events.lock().last().cloned().expect("recorded"); + assert_eq!(got.method, "histogram", "histogram option ignored"); + assert_eq!(got.name, "dialcache.get.duration", "default namespace"); +} + +#[test] +fn datadog_rejects_invalid_namespaces_and_long_names() { + for namespace in ["1bad", "has-dash", "two..dots", ".lead", "trail."] { + let error = DatadogObserver::new( + RecordingClient::default(), + DatadogOptions::new(ObservationMetricType::Distribution).namespace(namespace), + ) + .err() + .unwrap_or_else(|| panic!("accepted namespace {namespace:?}")); + assert_eq!(error, DatadogError::InvalidNamespace(namespace.to_string())); + } + let explicitly_empty = DatadogObserver::new( + RecordingClient::default(), + DatadogOptions::new(ObservationMetricType::Distribution).namespace(""), + ); + assert_eq!( + explicitly_empty.err(), + Some(DatadogError::InvalidNamespace(String::new())), + "accepted explicitly empty namespace" + ); + // The longest suffix, "stale_recovery.value_age", is 24 characters; with + // the dot a 175-character namespace reaches exactly 200 and 176 exceeds it. + let longest = "stale_recovery.value_age"; + assert_eq!(longest.len(), 24); + let fits = "a".repeat(175); + DatadogObserver::new( + RecordingClient::default(), + DatadogOptions::new(ObservationMetricType::Distribution).namespace(fits.clone()), + ) + .expect("200-character name fits"); + let overflow = "a".repeat(176); + let error = DatadogObserver::new( + RecordingClient::default(), + DatadogOptions::new(ObservationMetricType::Distribution).namespace(overflow.clone()), + ) + .expect_err("201-character name rejected"); + assert_eq!( + error, + DatadogError::MetricNameTooLong(format!("{overflow}.{longest}")) + ); + let huge = "a".repeat(201); + assert!(DatadogObserver::new( + RecordingClient::default(), + DatadogOptions::new(ObservationMetricType::Distribution).namespace(huge), + ) + .is_err()); +} + +#[cfg(feature = "prometheus")] +mod prometheus_exporter { + use super::*; + use dialcache::prometheus::{schemas, CollectorSchema}; + use dialcache::{PrometheusError, PrometheusObserver}; + use prometheus::proto::MetricType; + use prometheus::{Gauge, IntCounterVec, Opts, Registry}; + + struct Expected { + kind: &'static str, + name: &'static str, + help: &'static str, + labels: &'static [&'static str], + buckets: &'static [f64], + } + + const TIMER: &[f64] = &[ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]; + const SIZE: &[f64] = &[100.0, 1000.0, 10000.0, 100000.0, 1000000.0, 10000000.0]; + const AGE: &[f64] = &[ + 1.0, 5.0, 15.0, 60.0, 300.0, 900.0, 3600.0, 10800.0, 43200.0, 86400.0, 259200.0, 604800.0, + ]; + const LAYER: &[&str] = &["cache_namespace", "use_case", "key_type", "layer"]; + const OUTCOME: &[&str] = &["cache_namespace", "use_case", "key_type", "outcome"]; + + /// Copied from go/metrics_prometheus.go PrometheusCollectorSchemas, in order. + const EXPECTED: [Expected; 19] = [ + Expected { + kind: "disabled", + name: "dialcache_disabled_counter", + help: "Requests where DialCache skipped a cache layer.", + labels: &["cache_namespace", "use_case", "key_type", "layer", "reason"], + buckets: &[], + }, + Expected { + kind: "miss", + name: "dialcache_miss_counter", + help: "DialCache cache misses.", + labels: &["cache_namespace", "use_case", "key_type", "layer", "reason"], + buckets: &[], + }, + Expected { + kind: "request", + name: "dialcache_request_counter", + help: "Total DialCache cache-layer requests.", + labels: LAYER, + buckets: &[], + }, + Expected { + kind: "error", + name: "dialcache_error_counter", + help: "Errors during DialCache cache operations or fallback execution.", + labels: &[ + "cache_namespace", + "use_case", + "key_type", + "layer", + "error", + "in_fallback", + ], + buckets: &[], + }, + Expected { + kind: "invalidation", + name: "dialcache_invalidation_counter", + help: "DialCache invalidation calls by key type and layer.", + labels: &["cache_namespace", "key_type", "layer"], + buckets: &[], + }, + Expected { + kind: "coalesced", + name: "dialcache_coalesced_counter", + help: "DialCache requests coalesced onto in-flight work by sharing scope.", + labels: &["cache_namespace", "use_case", "key_type", "scope"], + buckets: &[], + }, + Expected { + kind: "shadowValidation", + name: "dialcache_shadow_validation_counter", + help: "Sampled DialCache Redis shadow-validation outcomes.", + labels: OUTCOME, + buckets: &[], + }, + Expected { + kind: "shadowValueAge", + name: "dialcache_shadow_value_age_histogram", + help: "Age in seconds of the validated Redis value at DialCache shadow verdict time.", + labels: OUTCOME, + buckets: AGE, + }, + Expected { + kind: "futureTimestampOffset", + name: "dialcache_future_timestamp_offset_histogram", + help: "Positive offset in seconds of Redis frames dated after the observing DialCache process clock.", + labels: LAYER, + buckets: &[ + 0.001, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 5.0, 15.0, 60.0, 300.0, + 900.0, 3600.0, 10800.0, 43200.0, + ], + }, + Expected { + kind: "staleRecovery", + name: "dialcache_stale_recovery_counter", + help: "DialCache stale-on-error Redis recovery outcomes.", + labels: OUTCOME, + buckets: &[], + }, + Expected { + kind: "staleRecoveryValueAge", + name: "dialcache_stale_recovery_value_age_histogram", + help: "Age in seconds of Redis values served by DialCache stale-on-error recovery.", + labels: OUTCOME, + buckets: AGE, + }, + Expected { + kind: "compression", + name: "dialcache_compression_counter", + help: "DialCache Redis payload compression and decompression outcomes.", + labels: &["cache_namespace", "use_case", "key_type", "layer", "outcome"], + buckets: &[], + }, + Expected { + kind: "get", + name: "dialcache_get_timer", + help: "DialCache cache get latency in seconds.", + labels: LAYER, + buckets: TIMER, + }, + Expected { + kind: "fallback", + name: "dialcache_fallback_timer", + help: "Time DialCache waited for the fallback function in seconds.", + labels: LAYER, + buckets: TIMER, + }, + Expected { + kind: "serialization", + name: "dialcache_serialization_timer", + help: "DialCache serialization latency in seconds.", + labels: &["cache_namespace", "use_case", "key_type", "layer", "operation"], + buckets: TIMER, + }, + Expected { + kind: "size", + name: "dialcache_size_histogram", + help: "Serialized DialCache value sizes in bytes.", + labels: LAYER, + buckets: SIZE, + }, + Expected { + kind: "storedSize", + name: "dialcache_stored_size_histogram", + help: "Stored DialCache payload sizes in bytes, after compression and escaping.", + labels: LAYER, + buckets: SIZE, + }, + Expected { + kind: "compressionRatio", + name: "dialcache_compression_ratio_histogram", + help: "Compressed-to-original DialCache payload size ratio for compressed writes.", + labels: LAYER, + buckets: &[0.05, 0.1, 0.2, 0.3, 0.5, 0.7, 0.9, 1.0], + }, + Expected { + kind: "compressionDuration", + name: "dialcache_compression_timer", + help: "DialCache payload compression and decompression latency in seconds.", + labels: &["cache_namespace", "use_case", "key_type", "layer", "operation"], + buckets: TIMER, + }, + ]; + + /// Port of TestPrometheusWireSchemaMatchesTypeScriptBinding against the + /// Go table, which that Go test pins to src/prometheus.ts. + #[test] + fn wire_schema_matches_the_reference_table() { + for prefix in ["", "svc_"] { + let actual: Vec = schemas(prefix); + assert_eq!(actual.len(), 19); + for (schema, expected) in actual.iter().zip(EXPECTED.iter()) { + assert_eq!(schema.kind.as_str(), expected.kind); + assert_eq!(schema.name, format!("{prefix}{}", expected.name)); + assert_eq!(schema.help, expected.help, "{} help drift", expected.name); + assert_eq!( + schema.labels, expected.labels, + "{} label order drift", + expected.name + ); + assert_eq!( + schema.buckets, expected.buckets, + "{} buckets drift", + expected.name + ); + assert_eq!(schema.is_counter(), expected.buckets.is_empty()); + } + } + let registry = Registry::new(); + let observer = PrometheusObserver::new(®istry, "svc_").expect("fresh registry"); + assert!(observer.observes_shadow_outcomes()); + assert_eq!(observer.schemas(), schemas("svc_")); + } + + /// The registered collectors expose the schema on a scrape: every kind + /// produces one family with the schema's name, help, type, label names + /// and bucket bounds. + #[test] + fn scrape_exposes_names_help_labels_and_buckets() { + let registry = Registry::new(); + let observer = PrometheusObserver::new(®istry, "scrape_").expect("fresh registry"); + for kind in MetricKind::ALL { + observer.observe(&metric_test_event(kind)); + } + let families = registry.gather(); + assert_eq!(families.len(), 19); + for expected in EXPECTED.iter() { + let name = format!("scrape_{}", expected.name); + let family = families + .iter() + .find(|f| f.name() == name) + .unwrap_or_else(|| panic!("{name} not scraped")); + assert_eq!(family.help(), expected.help); + let metric = &family.get_metric()[0]; + // The exposition sorts label pairs by name; wire order is pinned + // by the schema test above. + let mut label_names: Vec<&str> = metric.get_label().iter().map(|l| l.name()).collect(); + label_names.sort_unstable(); + let mut expected_labels = expected.labels.to_vec(); + expected_labels.sort_unstable(); + assert_eq!(label_names, expected_labels, "{name}"); + for label in metric.get_label() { + assert!(!label.value().contains(NEVER_EXPORTED)); + } + if expected.buckets.is_empty() { + assert_eq!(family.type_(), MetricType::COUNTER, "{name}"); + assert_eq!(metric.get_counter().value(), 1.0, "{name}"); + } else { + assert_eq!(family.type_(), MetricType::HISTOGRAM, "{name}"); + let histogram = metric.get_histogram(); + let bounds: Vec = histogram + .get_bucket() + .iter() + .map(|b| b.upper_bound()) + .collect(); + assert_eq!(bounds, expected.buckets, "{name}"); + assert_eq!(histogram.sample_count(), 1); + } + } + } + + /// Port of TestPrometheusReuseAndConflictIsolation. + #[test] + fn reuse_and_conflict_isolation() { + let registry = Registry::new(); + let first = PrometheusObserver::new(®istry, "test_").expect("first"); + let second = PrometheusObserver::new(®istry, "test_").expect("second reuses"); + first.observe(&metric_test_event(MetricKind::Request)); + second.observe(&metric_test_event(MetricKind::Request)); + first.observe(&metric_test_event(MetricKind::Get)); + let (mut saw_counter, mut saw_histogram) = (false, false); + for family in registry.gather() { + match family.name() { + "test_dialcache_request_counter" => { + saw_counter = true; + assert_eq!( + family.get_metric()[0].get_counter().value(), + 2.0, + "compatible observer did not reuse counter" + ); + } + "test_dialcache_get_timer" => { + saw_histogram = true; + let histogram = family.get_metric()[0].get_histogram(); + assert_eq!(histogram.sample_count(), 1); + assert_eq!(histogram.sample_sum(), 0.25); + assert_eq!(histogram.get_bucket().len(), 12); + } + _ => {} + } + } + assert!(saw_counter && saw_histogram, "metrics not exported"); + + // Reuse survives dropping every earlier observer. + drop(first); + drop(second); + let third = PrometheusObserver::new(®istry, "test_").expect("third reuses"); + third.observe(&metric_test_event(MetricKind::Request)); + let requests = registry + .gather() + .into_iter() + .find(|f| f.name() == "test_dialcache_request_counter") + .expect("request family"); + assert_eq!(requests.get_metric()[0].get_counter().value(), 3.0); + + // Another prefix on the same registry is an independent group. + let other = PrometheusObserver::new(®istry, "other_").expect("other prefix"); + other.observe(&metric_test_event(MetricKind::Request)); + let names: Vec = registry + .gather() + .iter() + .map(|f| f.name().to_string()) + .collect(); + assert!(names.contains(&"other_dialcache_request_counter".to_string())); + assert!(names.contains(&"test_dialcache_request_counter".to_string())); + + let conflict = Registry::new(); + conflict + .register(Box::new( + Gauge::new("dialcache_request_counter", "incompatible").expect("gauge"), + )) + .expect("gauge registered"); + let error = + PrometheusObserver::new(&conflict, "").expect_err("accepted conflicting collector"); + match &error { + PrometheusError::Conflict { name, .. } => { + assert_eq!(name, "dialcache_request_counter") + } + other => panic!("unexpected error {other}"), + } + assert!(error.to_string().contains("unique prefix or registry")); + // Only the gauge has an observed series... + assert_eq!( + conflict.gather().len(), + 1, + "failed observer partially registered collectors" + ); + // ...and the collectors registered before the conflict were rolled + // back, so their names are free again. + let disabled = schemas("") + .into_iter() + .find(|s| s.kind == MetricKind::Disabled) + .expect("disabled schema"); + conflict + .register(Box::new( + IntCounterVec::new(Opts::new(disabled.name, disabled.help), disabled.labels) + .expect("counter"), + )) + .expect("rolled-back collector name is free"); + } +} + +mod isolation { + use super::*; + use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; + use dialcache::{DialCache, KeySpec, Policy}; + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct PanickingObserver { + calls: Arc, + } + + impl Observer for PanickingObserver { + fn observe(&self, _event: &Event) { + self.calls.fetch_add(1, Ordering::SeqCst); + panic!("exporter failure"); + } + fn observes_shadow_outcomes(&self) -> bool { + panic!("exporter failure"); + } + } + + /// A panicking observer never changes the result of a cached call. + #[test] + fn panicking_observer_does_not_change_cached_results() { + let calls = Arc::new(AtomicUsize::new(0)); + let mut exec = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(exec.clock.clone()) + .runtime_arc(exec.runtime.clone()) + .observer(PanickingObserver { + calls: calls.clone(), + }) + .build() + .expect("configuration"); + let source_calls = Arc::new(AtomicUsize::new(0)); + let counting = source_calls.clone(); + let lookup = cache + .use_case::("item", "lookup") + .policy(Policy::default().local_ttl_sec(60)) + .key(|id: &i64| KeySpec::new(id.to_string())) + .source(move |_scope, id: i64| { + let counting = counting.clone(); + async move { + counting.fetch_add(1, Ordering::SeqCst); + Ok(id * 10) + } + }) + .register() + .expect("use case"); + + let first = { + let cache = cache.clone(); + let lookup = lookup.clone(); + exec.block_on(async move { + cache + .enable(|scope| async move { lookup.get(&scope, 7).await }) + .await + }) + }; + assert_eq!(*first.expect("source value survives observer panic"), 70); + let second = { + let cache = cache.clone(); + let lookup = lookup.clone(); + exec.block_on(async move { + cache + .enable(|scope| async move { lookup.get(&scope, 7).await }) + .await + }) + }; + assert_eq!(*second.expect("cached value survives observer panic"), 70); + assert_eq!( + source_calls.load(Ordering::SeqCst), + 1, + "second call was served from the local layer" + ); + assert!( + calls.load(Ordering::SeqCst) >= 2, + "the observer received the request events" + ); + } +} From f6e3c6955810ecdfe2d069c1650f8bf26c280280 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 01:12:04 -0700 Subject: [PATCH 15/31] fix(formal): replay the Rust port in exploration and name it in the formal guides --- formal/PORTING.md | 6 +++--- formal/README.md | 10 ++++----- formal/explore.mjs | 28 +++++++++++++++++++------ formal/go-parity.json | 4 ++-- formal/source-audit.json | 6 +++--- test/formal-exploration.test.ts | 36 ++++++++++++++++++++++----------- test/formal-rust-replay.test.ts | 3 ++- 7 files changed, 61 insertions(+), 32 deletions(-) diff --git a/formal/PORTING.md b/formal/PORTING.md index e97c6e87..faa2612c 100644 --- a/formal/PORTING.md +++ b/formal/PORTING.md @@ -452,13 +452,13 @@ replay does. The TypeScript suite runs the same evaluator over the histories it parsed for replay and writes no evidence. The Go replay also checks that every required label names at least one history of the bound corpus. -## Current limitations for a third port +## Current limitations for a further port The `causally-ready-v1` settlement contract is defined in prose, in the [trace and observation contract](#trace-and-observation-contract) and the `Settle` row above; no machine-checkable definition exists. Its executable -controls are the TypeScript and Go no-settle tests, each written against its -own driver, so a third port writes its own the same way. +controls are the TypeScript, Go and Rust no-settle tests, each written against +its own driver, so a further port writes its own the same way. Node 24 is required as test tooling: the coordinator and the witness evaluator are Node scripts that a port's test run spawns, and the completion checker and diff --git a/formal/README.md b/formal/README.md index a9ffd068..86e6a715 100644 --- a/formal/README.md +++ b/formal/README.md @@ -1,7 +1,7 @@ # Executable DialCache specification -Quint defines the portable contracts that TypeScript, Go and future ports must -preserve. Native drivers execute external commands against the real libraries; +Quint defines the portable contracts that TypeScript, Go, Rust and future ports +must preserve. Native drivers execute external commands against the real libraries; generated expectations stay in the test coordinator. ## Start with your task @@ -61,7 +61,7 @@ The verification models emphasize individual ownership or safety boundaries: | [dialcache-shadow-validation.qnt](./dialcache-shadow-validation.qnt) | Diagnostic C0/source/C1 work and fills | | [dialcache-redis-protocol.qnt](./dialcache-redis-protocol.qnt) | Frame/fence validation order | -Conformance profiles expose external commands that both language drivers replay: +Conformance profiles expose external commands that every language driver replays: | Profile | Behavior and interactions | | --- | --- | @@ -115,7 +115,7 @@ downloads its pinned solver archive. ```sh make help # Targets and prerequisites. make check # Native checks, package, docs and inventories. -make smoke # Committed Quint-derived histories in both ports. +make smoke # Committed Quint-derived histories in every port. make formal # Rust model checks, full corpus and both-port completion. make differential # Replay composed profiles' reference corpus through the working tree. make model-check # Separate finite symbolic checks; Java 21 and tar required. @@ -148,7 +148,7 @@ keeps a separate source snapshot, seed, corpus and diagnostic replay evidence. S [VALIDATION.md](./VALIDATION.md) for CI policy and report interpretation. Scheduled named public-action Quint regressions exercise their declared -boundaries independently of sampling. Both ports replay those histories and the +boundaries independently of sampling. Every port replays those histories and the complete sampled corpus; required witness coverage is checked across their union. A model regression reaches implementations only when registered for replay in `execution.json`, and the manifest validator requires every diff --git a/formal/explore.mjs b/formal/explore.mjs index 3ae39ebf..34d975dd 100644 --- a/formal/explore.mjs +++ b/formal/explore.mjs @@ -7,13 +7,14 @@ import { cleanEnvironment, executeSteps, validationPlan } from './validation.mjs import { nativeBinding } from './conformance-bindings.mjs'; import { parseTypeScriptReport } from './conformance-adapters.mjs'; import { checkGoReplay } from './check-go-replay.mjs'; +import { checkRustReplay } from './check-rust-replay.mjs'; import { canonicalSeed, reportFileName } from './witnesses.mjs'; const root = fileURLToPath(new URL('../', import.meta.url)); const hash = value => createHash('sha256').update(value).digest('hex'); const inside = (directory, path) => path.startsWith(directory + sep); -const reportPaths = { typescript: '.formal-traces/ts-replay.json', go: '.formal-traces/go-replay.jsonl' }; -const contextPaths = { typescript: '.formal-traces/ts-context.json', go: '.formal-traces/go-context.json' }; +const reportPaths = { typescript: '.formal-traces/ts-replay.json', go: '.formal-traces/go-replay.jsonl', rust: '.formal-traces/rust-replay.jsonl' }; +const contextPaths = { typescript: '.formal-traces/ts-context.json', go: '.formal-traces/go-context.json', rust: '.formal-traces/rust-context.json' }; export function explorationSeed(value = `0x${randomBytes(8).toString('hex')}`) { try { return canonicalSeed(value); } @@ -27,13 +28,13 @@ export function explorationPlan(directory, seed, options = {}) { const normalized = explorationSeed(seed); return validationPlan('formal', { ...options, directory }).flatMap(step => { const script = step.args?.[0]; - if (step.remove || ['formal/conformance-adapters.mjs', 'formal/check-go-replay.mjs'].includes(script) + if (step.remove || ['formal/conformance-adapters.mjs', 'formal/check-go-replay.mjs', 'formal/check-rust-replay.mjs'].includes(script) || script === 'formal/conformance.mjs' && step.args[1] === 'check') return []; if (script === 'formal/conformance.mjs' && step.args[1] === 'prepare') { return [{ label: `Prepare exploratory ${step.args[2]} context`, explorationContext: step.args[2] }]; } if (script === 'formal/run-models.mjs') return [{ ...step, env: { ...step.env, QUINT_SEED: normalized } }]; - if (step.env?.DIALCACHE_MBT_TRACE_DIR) return [{ ...step, nativeReport: step.command === 'go' ? 'go' : 'typescript' }]; + if (step.env?.DIALCACHE_MBT_TRACE_DIR) return [{ ...step, nativeReport: step.command === 'go' ? 'go' : step.command === 'cargo' ? 'rust' : 'typescript' }]; // A seed's missing witness is classified by both native reports. The shared // evaluator runs before either replay and still writes evidence for complete // profiles; its exit status must not stop either port from executing that @@ -155,6 +156,21 @@ export function nativeExplorationResult(language, text, context, directory, pack required: inventory.map(entry => ({ name: nativeBinding(entry, language), category: entry.category })) }; checkGoReplay(events.map(event => JSON.stringify(event.Action === 'fail' ? { ...event, Action: 'pass' } : event)).join('\n'), native); startedAt = Date.parse(events[0].Time); finishedAt = Date.parse(events.at(-1).Time); + } else if (language === 'rust') { + // The Rust report names cases by inventory id; a failed case record is the + // native counterexample. The all-passed copy reuses the strict report gate. + const records = text.trim().split('\n').map(line => JSON.parse(line)); + const cases = new Map(inventory.map(entry => [nativeBinding(entry, language), entry])); + for (const record of records) { + if (record.kind !== 'case' || record.status !== 'failed') continue; + const entry = cases.get(record.id); + if (entry) failed.push(entry); else otherFailures.push(record.id); + } + const finish = records.at(-1); + if (finish?.kind !== 'finish' || (finish.status === 'failed') !== (failed.length + otherFailures.length > 0)) throw new Error('Rust report status disagrees with its case records.'); + checkRustReplay(records.map(record => JSON.stringify(record.kind === 'case' ? { ...record, status: 'passed', message: undefined } + : record.kind === 'finish' ? { ...record, status: 'passed', failed: 0 } : record)).join('\n'), inventory); + startedAt = records[0]?.startedAt; finishedAt = finish.finishedAt; } else throw new Error('Unsupported exploratory port.'); if (!Number.isFinite(startedAt) || !Number.isFinite(finishedAt) || startedAt < context.createdAt || finishedAt < startedAt || finishedAt > Date.now() + 60_000) throw new Error('Stale or invalid native report timestamps.'); @@ -324,9 +340,9 @@ async function executeExploration(seed, { directory = root, environment = proces }); verifyHashes(workspace, [report.sources]); report.sourcesUnchanged = true; - if (report.native.map(result => result.language).sort().join() !== 'go,typescript' + if (report.native.map(result => result.language).sort().join() !== 'go,rust,typescript' || report.native.some(result => !['passed', 'native-failure', 'witness-check-failure'].includes(result.status))) { - throw new Error('Exploration did not finish both native ports.'); + throw new Error('Exploration did not finish every native port.'); } // The witness step is tolerated so both ports replay, but its baseline // gate still decides the outcome afterwards: a fresh seed whose sampled diff --git a/formal/go-parity.json b/formal/go-parity.json index 05088110..37e43d12 100644 --- a/formal/go-parity.json +++ b/formal/go-parity.json @@ -10,7 +10,7 @@ "inputs": { "semanticCasesSha256": "7054107347f57a452e17de99f74310553b7af4350363c674e70ff8184d0610be", "executionSha256": "feee09a3487deb2694f8784b7e8d2d0ee28050dae03ac43ac2613608ce5ab1c6", - "sourceAuditSha256": "09e9981795420d50cbd30d7f56ef4aeec750724961e794c2dd5526199dc91446", + "sourceAuditSha256": "ca0fcbb9b190ccfcb101d427d61794abf62296d1618c28156e8301bbb52fe0d1", "quintCaseAuditSha256": "dcb576948a68a066e43259d4bee2b3cdadbe82e657ae9af99e4b21ed7b95dbb3", "featureCoverageSha256": "56eedfa865682db6ad8e73b8673c7dac1c30d301e9bce3390246d6994aba9dcb", "coverageWitnessesSha256": "8fc8ab123150fdec059f5b75f84172d39af80d08d7a4211fa88c63c5a3f2874b", @@ -18305,7 +18305,7 @@ ], "reviewedTestAndDocumentationAudit": { "path": "formal/source-audit.json", - "sha256": "09e9981795420d50cbd30d7f56ef4aeec750724961e794c2dd5526199dc91446", + "sha256": "ca0fcbb9b190ccfcb101d427d61794abf62296d1618c28156e8301bbb52fe0d1", "sourceFiles": 44, "status": "reviewed-applicability-not-execution", "limitations": "These rows review Go applicability for the public documentation and ordinary test files in source-audit.json sources, inheriting their C/W/E/B/X classifications. They map subjects to native artifacts without claiming one Go test per TypeScript assertion or turning documentation counts into executed coverage. The same audit hash also binds reviewedGuides for formal and Go documentation; those guide reviews are freshness/scope records, not additional behavioral applicability rows. Per-case Quint and replay evidence is recorded separately.", diff --git a/formal/source-audit.json b/formal/source-audit.json index ba477d2e..755db4ad 100644 --- a/formal/source-audit.json +++ b/formal/source-audit.json @@ -1287,7 +1287,7 @@ }, { "path": "formal/PORTING.md", - "sha256": "ae7780bea1efc026232b3b48af92bce6f40df9591d6fc633c9c6130cea1ebd5f", + "sha256": "9280203fd477f26f728e8334a8541a8755f0977c32913b7db08cea0256077c25", "entries": [ {"line":1,"title":"Implementing another DialCache port"}, {"line":14,"title":"What a port supplies"}, @@ -1297,7 +1297,7 @@ {"line":222,"title":"Generated fixtures and fast local tests"}, {"line":257,"title":"Complete corpus and reusable acceptance checks"}, {"line":365,"title":"Witness evidence"}, - {"line":455,"title":"Current limitations for a third port"}, + {"line":455,"title":"Current limitations for a further port"}, {"line":469,"title":"New-port acceptance"} ], "review": { @@ -1346,7 +1346,7 @@ }, { "path": "formal/README.md", - "sha256": "0317648114092eae71371cff05fe40d049be4f66f87002074fa1b6fef46f53a1", + "sha256": "408c23982e53874489d1aa804c7ece96b7f487cf7e0a4a07552fcf914ca015d7", "entries": [ {"line":1,"title":"Executable DialCache specification"}, {"line":7,"title":"Start with your task"}, diff --git a/test/formal-exploration.test.ts b/test/formal-exploration.test.ts index dc6c861d..e062b189 100644 --- a/test/formal-exploration.test.ts +++ b/test/formal-exploration.test.ts @@ -43,6 +43,16 @@ function tsReport(directory: string, failure?: string) { testResults: [{ name: `${directory}/test/formal-features.test.ts`, status: failure ? "failed" : "passed", startTime: 2, endTime: 3, message: "", assertionResults }] }; } +function rustReport(failure?: string) { + const records: Record[] = [{ schemaVersion: 1, kind: "start", implementation: "rust", startedAt: 10 }]; + for (const [index, entry] of inventory.entries()) { + const failed = entry.id === failure; + records.push({ kind: "case", id: nativeBinding(entry, "rust"), status: failed ? "failed" : "passed", startedAt: 11 + index, finishedAt: 12 + index, + ...(failed ? { message: "Observation mismatch\nexpected: 1\nactual: 2" } : {}) }); + } + records.push({ kind: "finish", status: failure ? "failed" : "passed", finishedAt: 20, cases: inventory.length, failed: failure ? 1 : 0 }); + return records.map(record => JSON.stringify(record)).join("\n"); +} function goReport(failure?: string) { const events: Record[] = []; const event = (Action: string, Test?: string) => events.push({ Action, ...(Test ? { Test } : {}), Package: packageName, Time: new Date(10 + events.length).toISOString() }); @@ -92,7 +102,7 @@ function savedFixture(directory: string) { writeFileSync(options.directory + '/.formal-traces/saved-runner.json', JSON.stringify(plan)); // A saved run's evaluator also has to leave a completed witness report behind. writeFileSync(options.directory + '/.formal-traces/witness-report.json', ${JSON.stringify(JSON.stringify(completedWitnessReport("0x2a", ["effects"])))}); - return [{ language: 'typescript', status: 'passed' }, { language: 'go', status: 'passed' }]; + return [{ language: 'typescript', status: 'passed' }, { language: 'go', status: 'passed' }, { language: 'rust', status: 'passed' }]; }`, "formal/validation.mjs": `import { mkdirSync, writeFileSync } from 'node:fs'; export function checkPrerequisites(target, { directory }) { @@ -131,10 +141,11 @@ describe("isolated exploratory validation", () => { ]); for (const step of seeded) expect(step.env?.QUINT_SEED).toBe("0x2a"); const replays = plan.filter(step => step.nativeReport); - expect(replays.map(step => step.nativeReport)).toEqual(["typescript", "go"]); + expect(replays.map(step => step.nativeReport)).toEqual(["typescript", "go", "rust"]); for (const step of replays) expect(step.env?.DIALCACHE_FEATURE_TRACE_DIR).toBe(`${directory}/.formal-traces/features`); - expect(plan.filter(step => step.explorationContext).map(step => step.explorationContext)).toEqual(["typescript", "go"]); - expect(plan.some(step => step.args?.includes("formal/check-go-replay.mjs") || step.args?.includes("formal/conformance-adapters.mjs"))).toBe(false); + expect(plan.filter(step => step.explorationContext).map(step => step.explorationContext)).toEqual(["typescript", "go", "rust"]); + expect(plan.some(step => step.args?.includes("formal/check-go-replay.mjs") || step.args?.includes("formal/check-rust-replay.mjs") + || step.args?.includes("formal/conformance-adapters.mjs"))).toBe(false); expect(plan.some(step => step.args?.[0] === "formal/conformance.mjs" && step.args[1] === "check")).toBe(false); }); @@ -161,8 +172,9 @@ describe("isolated exploratory validation", () => { } finally { rmSync(directory, { recursive: true, force: true }); } }); - it.each(["typescript", "go"])("classifies exact %s witness leaves separately from replay failures", language => { - const native = (failure?: string) => language === "typescript" ? JSON.stringify(tsReport("/snapshot", failure)) : goReport(failure); + it.each(["typescript", "go", "rust"])("classifies exact %s witness leaves separately from replay failures", language => { + const native = (failure?: string) => language === "typescript" ? JSON.stringify(tsReport("/snapshot", failure)) + : language === "go" ? goReport(failure) : rustReport(failure); expect(nativeExplorationResult(language, native(), context(language), "/snapshot", packageName).status).toBe("passed"); expect(nativeExplorationResult(language, native("witness/recovery"), context(language), "/snapshot", packageName)).toMatchObject({ status: "witness-check-failure", witnessFailures: ["witness/recovery"], caseFailures: [], @@ -239,7 +251,7 @@ describe("isolated exploratory validation", () => { await expect(explore("42", { directory, run: async () => { const output = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!); writeFileSync(join(output, "workspace/rule.qnt"), "changed"); - return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }]; + return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }, { language: "rust", status: "passed" }]; } })).rejects.toThrow(/changed/); const output = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!); expect(readFileSync(join(directory, "rule.qnt"), "utf8")).toBe("original"); @@ -256,12 +268,12 @@ describe("isolated exploratory validation", () => { writeFileSync(join(directory, ".gitignore"), ".formal-traces/\n"); writeWitnessInventory(directory, selectedProfiles("all")); await expect(explore("42", { directory, run: async () => [ - { language: "typescript", status: "witness-check-failure" }, { language: "go", status: "witness-check-failure" }, + { language: "typescript", status: "witness-check-failure" }, { language: "go", status: "witness-check-failure" }, { language: "rust", status: "witness-check-failure" }, ] })).rejects.toThrow(/witness-check-failure/); const output = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!); expect(JSON.parse(readFileSync(join(output, "report.json"), "utf8"))).toMatchObject({ kind: "exploration", acceptance: false, status: "witness-check-failure", sourcesUnchanged: true, - native: [{ language: "typescript" }, { language: "go" }], + native: [{ language: "typescript" }, { language: "go" }, { language: "rust" }], }); expect(existsSync(join(output, "workspace/node_modules"))).toBe(false); } finally { rmSync(directory, { recursive: true, force: true }); } @@ -337,7 +349,7 @@ describe("isolated exploratory validation", () => { mkdirSync(join(workspace, ".formal-traces"), { recursive: true }); writeFileSync(join(workspace, ".formal-traces/witness-report.json"), JSON.stringify(witnesses)); replays = 2; - return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }]; + return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }, { language: "rust", status: "passed" }]; } })).rejects.toThrow(/coverage-gate-failure[\s\S]*reply:13 reached by 1 sampled histories/); expect(replays).toBe(2); const output = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!); @@ -363,7 +375,7 @@ describe("isolated exploratory validation", () => { const workspace = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!, "workspace"); mkdirSync(join(workspace, ".formal-traces"), { recursive: true }); if (contents !== undefined) writeFileSync(join(workspace, ".formal-traces/witness-report.json"), contents); - return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }]; + return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }, { language: "rust", status: "passed" }]; } })).rejects.toThrow(message); const output = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!); expect(JSON.parse(readFileSync(join(output, "report.json"), "utf8")).status).toBe("infrastructure-failure"); @@ -390,7 +402,7 @@ describe("isolated exploratory validation", () => { const workspace = join(directory, ".formal-traces/exploration", readdirSync(join(directory, ".formal-traces/exploration"))[0]!, "workspace"); mkdirSync(join(workspace, ".formal-traces"), { recursive: true }); writeFileSync(join(workspace, ".formal-traces/witness-report.json"), JSON.stringify(witnesses)); - return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }]; + return [{ language: "typescript", status: "passed" }, { language: "go", status: "passed" }, { language: "rust", status: "passed" }]; } }); expect(JSON.parse(readFileSync(join(output, "report.json"), "utf8"))).toMatchObject({ status: "passed", witnesses }); } finally { rmSync(directory, { recursive: true, force: true }); } diff --git a/test/formal-rust-replay.test.ts b/test/formal-rust-replay.test.ts index 033cfacb..f733ad39 100644 --- a/test/formal-rust-replay.test.ts +++ b/test/formal-rust-replay.test.ts @@ -86,7 +86,8 @@ describe("Rust replay report gate", () => { expect(() => checkRustReplay(withCases(records => { records.shift(); }), inventory)).toThrow(/precedes the start record/); expect(() => checkRustReplay(withCases(records => { records.splice(1, 0, start()); }), inventory)).toThrow(/Duplicate or misplaced Rust start record/); expect(() => checkRustReplay(withCases(records => { records[0] = { ...start(), implementation: "go" }; }), inventory)).toThrow(/Unsupported Rust start record/); - expect(() => checkRustReplay(encode(completed()).replace("\n", "\n{not json\n"), inventory)).toThrow("Invalid Rust JSON record at line 2"); + const [firstLine, ...rest] = encode(completed()).split("\n"); + expect(() => checkRustReplay([firstLine, "{not json", ...rest].join("\n"), inventory)).toThrow("Invalid Rust JSON record at line 2"); expect(() => checkRustReplay(withCases(records => { records[1] = { kind: "note", id: "x" }; }), inventory)).toThrow(/Unsupported Rust replay record kind: note/); expect(() => checkRustReplay("", inventory)).toThrow(/Empty Rust replay report/); }); From b055a1c2e6df0ca51077ca9c0d15b0f2394ff1d4 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 01:13:35 -0700 Subject: [PATCH 16/31] feat(rust): add the redis crate Remote adapter with real-server and invalidation vector tests RedisAdapter implements Remote over a caller-owned redis crate handle (ConnectionManager, MultiplexedConnection or ClusterConnection) through a small RedisConnection trait. Wire behavior matches Go and TypeScript: untracked GET, tracked MGET routed explicitly to the slot primary on a cluster, exactly one SET ... PX per write, EVALSHA with one EVAL retry carrying identical arguments for invalidation, strict reply validation that is never retried, and cooperative read cancellation. The Lua transition is byte-identical to go/redis_adapter.go and pinned by a unit test; the redis dependency gains the connection-manager feature, which the crate requires for ConnectionManager. tests/redis_integration.rs starts Redis 6.2, Redis 7 and Valkey 8 standalone servers plus a single-node Redis 7 cluster with Docker when DIALCACHE_RUST_INTEGRATION=1 and replays frame round trips, watermark fencing, SCRIPT FLUSH recovery and all 337 invalidation vectors through tests/formal/invalidation_vectors.rs. --- rust/Cargo.lock | 32 + rust/Cargo.toml | 2 +- rust/src/lib.rs | 7 + rust/src/redis.rs | 883 ++++++++++++++++++++++ rust/tests/formal/invalidation_vectors.rs | 327 ++++++++ rust/tests/redis_integration.rs | 486 ++++++++++++ 6 files changed, 1736 insertions(+), 1 deletion(-) create mode 100644 rust/src/redis.rs create mode 100644 rust/tests/formal/invalidation_vectors.rs create mode 100644 rust/tests/redis_integration.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index e9381413..8a26a97c 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -8,6 +8,15 @@ version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +[[package]] +name = "arc-swap" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b" +dependencies = [ + "rustversion", +] + [[package]] name = "arcstr" version = "1.2.0" @@ -31,6 +40,15 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "backon" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" +dependencies = [ + "fastrand", +] + [[package]] name = "bitflags" version = "2.13.2" @@ -204,6 +222,12 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + [[package]] name = "find-msvc-tools" version = "0.1.13" @@ -708,8 +732,10 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2acbc41a996f7652b2ddd9dfd98cc4ff602cfd742ae35382f07f608405ab50ed" dependencies = [ + "arc-swap", "arcstr", "async-lock", + "backon", "bytes", "cfg-if", "combine", @@ -740,6 +766,12 @@ dependencies = [ "bitflags", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + [[package]] name = "ryu" version = "1.0.23" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 039f909e..91a5e42a 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -29,7 +29,7 @@ zstd-safe = "7" thiserror = "2" log = "0.4" tokio = { version = "1", features = ["rt", "time"], optional = true } -redis = { version = "1", features = ["tokio-comp", "cluster-async", "script"], optional = true } +redis = { version = "1", features = ["tokio-comp", "cluster-async", "connection-manager", "script"], optional = true } sha1 = { version = "0.11", optional = true } hex = { version = "0.4", optional = true } prometheus = { version = "0.14", optional = true } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 154a36ad..8d8828d1 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -25,6 +25,8 @@ pub mod operation; pub mod policy; pub mod preview; pub mod protocol; +#[cfg(feature = "redis")] +pub mod redis; pub mod remote; pub mod runtime; mod scope; @@ -48,6 +50,11 @@ pub use operation::{Comparator, Operation, Preview, RecoveryPredicate, SourceBud pub use policy::{ Policy, PolicyDefaults, PolicyError, ResolvedPolicy, RuntimePolicy, ShadowPolicy, }; +#[cfg(feature = "redis")] +pub use redis::{ + invalidation_script_sha1, RedisAdapter, RedisConnection, RedisProtocolError, + RedisReadCancelled, INVALIDATION_SCRIPT, +}; pub use remote::{ Frame, InvalidateRequest, MissReason, ReadContext, ReadRequest, ReadResult, Remote, WriteRequest, diff --git a/rust/src/redis.rs b/rust/src/redis.rs new file mode 100644 index 00000000..ce1470ce --- /dev/null +++ b/rust/src/redis.rs @@ -0,0 +1,883 @@ +//! Production [`Remote`] adapter over the [`redis`] crate. +//! +//! [`RedisAdapter`] borrows a connected, caller-owned connection handle: a +//! [`ConnectionManager`] for standalone or Sentinel-managed servers, a +//! [`ClusterConnection`] for Redis Cluster, or a plain +//! [`MultiplexedConnection`]. It never connects, reconnects, drains or closes +//! the handle and never changes its options. The caller remains responsible +//! for: +//! +//! - **Finite budgets.** Configure connection, response and retry limits on +//! the handle (`ConnectionManagerConfig::set_response_timeout`, +//! `ClusterClientBuilder::response_timeout`, ...). The cache bounds its own +//! wait for reads and honors cooperative cancellation, but cancellation +//! after dispatch cannot prove a command did not execute. +//! - **Primaries for tracked reads.** On a cluster handle the adapter routes +//! every tracked `MGET` explicitly to the slot primary, so replica lag can +//! never hide an invalidation watermark even when the caller enabled +//! `read_from_replicas`. A standalone or Sentinel-managed handle must point +//! at the primary itself; the adapter cannot tell a replica apart. +//! - **Watermark retention.** Invalidation stores watermarks with a +//! retention of at least two hours that only widens. Configure eviction so +//! watermark keys are not evicted before tracked values age out; an +//! evicted watermark resurrects fenced values. +//! +//! Wire behavior matches the TypeScript and Go adapters exactly: an +//! untracked read is one `GET`; a tracked read is one `MGET value watermark` +//! on the primary; a write is exactly one native `SET key frame PX ttl`; an +//! invalidation is `EVALSHA` of [`INVALIDATION_SCRIPT`] with one `EVAL` +//! retry carrying identical arguments after any `EVALSHA` failure. Accepted +//! but invalid replies are [`RedisProtocolError`]s and are never retried. + +use std::fmt; + +use futures::future::{self, BoxFuture, Either, FutureExt}; +use redis::aio::{ConnectionLike, ConnectionManager, MultiplexedConnection}; +use redis::cluster_async::ClusterConnection; +use redis::cluster_routing::{Route, RoutingInfo, SingleNodeRoutingInfo, Slot, SlotAddr}; +use redis::{Cmd, RedisResult, Value}; +use sha1::{Digest, Sha1}; + +use crate::error::BoxError; +use crate::limits::{MAX_SAFE_INTEGER, MAX_SUPPORTED_DURATION_MS}; +use crate::protocol::{decode_frame, encode_frame, ProtocolError}; +use crate::remote::{ + InvalidateRequest, ReadContext, ReadRequest, ReadResult, Remote, WriteRequest, +}; + +/// The version-1 wire invalidation transition, byte-identical to the Go +/// `InvalidationScript` (a unit test pins this against `go/redis_adapter.go`). +/// TypeScript's `INVALIDATE_CACHE_SCRIPT` is the same Lua with different +/// whitespace and a comment, so its `EVALSHA` digest differs; every port +/// self-heals through the `EVAL` fallback, so mixed-language clusters need no +/// shared script cache. +/// +/// Redis Lua numbers exactly represent the accepted safe-integer domain. +/// Invalid arguments return before `GET`/repair; wrong-type keys alone are +/// repairable read failures. +pub const INVALIDATION_SCRIPT: &str = r#"local function parse_safe_integer(raw) + if not string.match(raw, "^%d+$") then return nil end + local value = tonumber(raw) + if not value or value > 9007199254740991 then return nil end + return value +end +local future_buffer_ms = parse_safe_integer(ARGV[1]) +if not future_buffer_ms or future_buffer_ms < 0 or future_buffer_ms > 31536000000 then + return redis.error_reply("ERR invalid DialCache future buffer") +end +local invalidated_at_ms = parse_safe_integer(ARGV[2]) +if not invalidated_at_ms or invalidated_at_ms > 9007199254740991 - future_buffer_ms then + return redis.error_reply("ERR invalid DialCache invalidatedAtMs") +end +local proposed_watermark = invalidated_at_ms + future_buffer_ms +local raw_watermark = redis.pcall("GET", KEYS[1]) +if type(raw_watermark) == "table" and raw_watermark.err then + if not string.match(raw_watermark.err, "^WRONGTYPE ") then return raw_watermark end + raw_watermark = false +end +local current_watermark = 0 +if raw_watermark then + local parsed_watermark = parse_safe_integer(raw_watermark) + if parsed_watermark then current_watermark = parsed_watermark end +end +local watermark = math.max(current_watermark, proposed_watermark) +local current_ttl_ms = -2 +if raw_watermark then current_ttl_ms = redis.call("PTTL", KEYS[1]) end +local desired_ttl_ms = math.max(7200000, watermark - invalidated_at_ms + 3600000 + 60000) +if current_ttl_ms > desired_ttl_ms then desired_ttl_ms = current_ttl_ms end +local encoded_watermark = string.format("%.0f", watermark) +if current_ttl_ms == -1 then + redis.call("SET", KEYS[1], encoded_watermark) +else + redis.call("SET", KEYS[1], encoded_watermark, "PX", desired_ttl_ms) +end +return 1"#; + +/// Lowercase hex SHA-1 of [`INVALIDATION_SCRIPT`], the `EVALSHA` argument. +pub fn invalidation_script_sha1() -> String { + hex::encode(Sha1::digest(INVALIDATION_SCRIPT.as_bytes())) +} + +/// A reply the server accepted but that violates the DialCache wire protocol. +/// +/// Such replies are never retried: the command executed and the adapter has +/// no way to tell what state the server holds. +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("Invalid DialCache Redis reply: {message}")] +pub struct RedisProtocolError { + pub message: String, +} + +impl RedisProtocolError { + fn new(message: impl Into) -> Self { + RedisProtocolError { + message: message.into(), + } + } +} + +/// A read stopped waiting because its [`ReadContext`] was cancelled. +/// +/// The command may still execute on the server; the cache's own deadline is +/// authoritative and this error only ends the adapter's wait. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("DialCache Redis read cancelled before a reply arrived")] +pub struct RedisReadCancelled; + +/// A cloneable, task-shareable handle that runs one Redis command. +/// +/// Implemented for [`ConnectionManager`], [`MultiplexedConnection`] and +/// [`ClusterConnection`]. Test doubles implement it to observe dispatched +/// arguments without a server. Every method borrows `&self`; implementations +/// clone the underlying handle per command, which the `redis` crate's handles +/// make cheap. +pub trait RedisConnection: Send + Sync + 'static { + /// Run one command with the handle's default routing. + fn run(&self, cmd: Cmd) -> BoxFuture<'_, RedisResult>; + + /// Run one command that reads `key`, on the node that owns the key's + /// slot as a primary. Non-cluster handles have exactly one node and use + /// their default routing. + fn run_on_primary(&self, key: &str, cmd: Cmd) -> BoxFuture<'_, RedisResult>; +} + +/// Lift a RESP3 error reply carried as a value into the error channel so +/// every caller sees one failure shape. +fn settle(reply: RedisResult) -> RedisResult { + match reply { + Ok(Value::ServerError(error)) => Err(error.into()), + other => other, + } +} + +fn run_cloned(connection: &C, cmd: Cmd) -> BoxFuture<'_, RedisResult> +where + C: ConnectionLike + Clone + Send + Sync, +{ + let mut connection = connection.clone(); + async move { settle(connection.req_packed_command(&cmd).await) }.boxed() +} + +impl RedisConnection for ConnectionManager { + fn run(&self, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + run_cloned(self, cmd) + } + + fn run_on_primary(&self, _key: &str, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + run_cloned(self, cmd) + } +} + +impl RedisConnection for MultiplexedConnection { + fn run(&self, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + run_cloned(self, cmd) + } + + fn run_on_primary(&self, _key: &str, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + run_cloned(self, cmd) + } +} + +impl RedisConnection for ClusterConnection { + fn run(&self, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + run_cloned(self, cmd) + } + + /// Selecting the slot primary is explicit here because a stale replica + /// could hide an invalidation fence when the client enabled replica reads. + fn run_on_primary(&self, key: &str, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + let route = Route::with_slot(Slot::for_key(key), SlotAddr::Master); + let routing = RoutingInfo::SingleNode(SingleNodeRoutingInfo::SpecificNode(route)); + let mut connection = self.clone(); + async move { settle(connection.route_command(cmd, routing).await) }.boxed() + } +} + +/// [`Remote`] over a caller-owned `redis` crate connection handle. +/// +/// See the [module documentation](self) for the caller's responsibilities. +#[derive(Clone)] +pub struct RedisAdapter { + connection: C, +} + +impl fmt::Debug for RedisAdapter { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RedisAdapter").finish_non_exhaustive() + } +} + +impl RedisAdapter { + /// Borrow `connection`. The adapter never connects or closes it. + pub fn new(connection: C) -> Self { + RedisAdapter { connection } + } + + /// The borrowed connection handle. + pub fn connection(&self) -> &C { + &self.connection + } + + /// The script's raw decimal boundary for out-of-band tooling and + /// conformance replay. Validation inside Lua precedes every mutation. + /// + /// `EVALSHA` runs first; any failure is retried exactly once as `EVAL` + /// with identical arguments, and a failed retry surfaces unmodified. The + /// transition is idempotent (the watermark only advances and its + /// retention only widens), so a duplicate run after an ambiguous failure + /// is harmless. A successful reply must be the integer `1`. + pub fn invalidate_decimal<'a>( + &'a self, + watermark_key: &'a str, + future_buffer_ms: &'a str, + invalidated_at_ms: &'a str, + ) -> BoxFuture<'a, Result<(), BoxError>> { + async move { + let script_command = |program: &str, script: &str| { + let mut cmd = redis::cmd(program); + cmd.arg(script) + .arg(1) + .arg(watermark_key) + .arg(future_buffer_ms) + .arg(invalidated_at_ms); + cmd + }; + let evalsha = script_command("EVALSHA", &invalidation_script_sha1()); + let reply = match self.connection.run(evalsha).await { + Ok(reply) => reply, + Err(_) => { + let eval = script_command("EVAL", INVALIDATION_SCRIPT); + self.connection.run(eval).await? + } + }; + validate_invalidation_reply(&reply)?; + Ok(()) + } + .boxed() + } +} + +/// One bulk-string reply position: `None` is absent. +type Bulk = Option>; + +/// One bulk-string reply position: `Nil` is absent, string replies are +/// content, anything else is a protocol violation. +fn bulk(reply: Value) -> Result { + match reply { + Value::Nil => Ok(None), + Value::BulkString(bytes) => Ok(Some(bytes)), + Value::SimpleString(text) => Ok(Some(text.into_bytes())), + other => Err(RedisProtocolError::new(format!( + "expected Redis bulk string, got {}", + value_kind(&other) + ))), + } +} + +/// A tracked read must return exactly two bulk values. +fn tracked_reply(reply: Value) -> Result<(Bulk, Bulk), RedisProtocolError> { + let Value::Array(items) = reply else { + return Err(RedisProtocolError::new( + "tracked read must return exactly two bulk values", + )); + }; + let [value, watermark]: [Value; 2] = items + .try_into() + .map_err(|_| RedisProtocolError::new("tracked read must return exactly two bulk values"))?; + Ok((bulk(value)?, bulk(watermark)?)) +} + +/// `SET` must answer `OK`. +pub fn validate_set_reply(reply: &Value) -> Result<(), RedisProtocolError> { + let ok = match reply { + Value::Okay => true, + Value::SimpleString(text) => text == "OK", + Value::BulkString(bytes) => bytes == b"OK", + _ => false, + }; + if ok { + Ok(()) + } else { + Err(RedisProtocolError::new( + "invalid Redis SET reply; expected OK", + )) + } +} + +/// The invalidation script must answer the integer `1`. +pub fn validate_invalidation_reply(reply: &Value) -> Result<(), RedisProtocolError> { + if matches!(reply, Value::Int(1)) { + Ok(()) + } else { + Err(RedisProtocolError::new( + "invalid Redis invalidation reply; expected integer 1", + )) + } +} + +fn value_kind(value: &Value) -> &'static str { + match value { + Value::Nil => "nil", + Value::Int(_) => "integer", + Value::BulkString(_) => "bulk string", + Value::Array(_) => "array", + Value::SimpleString(_) => "simple string", + Value::Okay => "OK", + Value::Map(_) => "map", + Value::Attribute { .. } => "attribute", + Value::Set(_) => "set", + Value::Double(_) => "double", + Value::Boolean(_) => "boolean", + Value::VerbatimString { .. } => "verbatim string", + Value::BigNumber(_) => "big number", + Value::Push { .. } => "push", + Value::ServerError(_) => "server error", + _ => "unknown", + } +} + +impl Remote for RedisAdapter { + fn read( + &self, + request: ReadRequest, + context: ReadContext, + ) -> BoxFuture<'_, Result> { + async move { + if context.cancel.is_cancelled() { + return Err(Box::new(RedisReadCancelled) as BoxError); + } + let tracked = request.watermark_key.is_some(); + let command = match &request.watermark_key { + None => { + let mut cmd = redis::cmd("GET"); + cmd.arg(&request.value_key); + self.connection.run(cmd) + } + Some(watermark_key) => { + let mut cmd = redis::cmd("MGET"); + cmd.arg(&request.value_key).arg(watermark_key); + self.connection.run_on_primary(&request.value_key, cmd) + } + }; + let reply = match future::select(command, context.cancel.cancelled()).await { + Either::Left((reply, _)) => reply?, + Either::Right(((), _)) => return Err(Box::new(RedisReadCancelled) as BoxError), + }; + let (value, watermark) = if tracked { + tracked_reply(reply)? + } else { + (bulk(reply)?, None) + }; + Ok(decode_frame( + value.as_deref(), + tracked, + watermark.as_deref(), + )?) + } + .boxed() + } + + /// Exactly one native `SET` of the complete frame. It neither reads nor + /// modifies the entity watermark; the frame's stamp is stored exactly. + fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + async move { + if request.ttl_ms == 0 || request.ttl_ms > MAX_SUPPORTED_DURATION_MS { + return Err(ProtocolError::InvalidDuration.into()); + } + let raw = encode_frame(&request.frame)?; + let mut cmd = redis::cmd("SET"); + cmd.arg(&request.value_key) + .arg(raw) + .arg("PX") + .arg(request.ttl_ms.to_string()); + let reply = self.connection.run(cmd).await?; + validate_set_reply(&reply)?; + Ok(()) + } + .boxed() + } + + /// The caller's clock sample stays stable through the `EVAL` retry, so + /// one logical invalidation proposes one watermark. + fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + async move { + if request.future_buffer_ms > MAX_SUPPORTED_DURATION_MS { + return Err(ProtocolError::InvalidDuration.into()); + } + if request.invalidated_at_ms > MAX_SAFE_INTEGER - request.future_buffer_ms { + return Err(ProtocolError::InvalidTimestamp.into()); + } + self.invalidate_decimal( + &request.watermark_key, + &request.future_buffer_ms.to_string(), + &request.invalidated_at_ms.to_string(), + ) + .await + } + .boxed() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cancel::CancelToken; + use crate::codec::Payload; + use crate::remote::{Frame, MissReason}; + use parking_lot::Mutex; + use std::collections::VecDeque; + use std::path::Path; + use std::sync::Arc; + use std::time::Duration; + + /// One recorded dispatch: whether it was pinned to a primary, and its + /// arguments including the command name. + #[derive(Debug, Clone, PartialEq, Eq)] + struct Call { + primary: bool, + args: Vec>, + } + + /// Replays scripted replies in order; a missing reply means "hang". + #[derive(Default)] + struct Scripted { + calls: Mutex>, + replies: Mutex>>, + } + + impl Scripted { + fn with(replies: Vec>) -> Arc { + Arc::new(Scripted { + calls: Mutex::new(Vec::new()), + replies: Mutex::new(replies.into_iter().collect()), + }) + } + + fn calls(&self) -> Vec { + self.calls.lock().clone() + } + + fn dispatch(&self, primary: bool, cmd: &Cmd) -> BoxFuture<'static, RedisResult> { + let args = cmd + .args_iter() + .map(|arg| match arg { + redis::Arg::Simple(bytes) => bytes.to_vec(), + _ => b"".to_vec(), + }) + .collect(); + self.calls.lock().push(Call { primary, args }); + match self.replies.lock().pop_front() { + Some(reply) => future::ready(reply).boxed(), + None => future::pending().boxed(), + } + } + } + + impl RedisConnection for Arc { + fn run(&self, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + self.dispatch(false, &cmd) + } + + fn run_on_primary(&self, _key: &str, cmd: Cmd) -> BoxFuture<'_, RedisResult> { + self.dispatch(true, &cmd) + } + } + + fn failure() -> RedisResult { + Err(redis::RedisError::from(( + redis::ErrorKind::Io, + "ambiguous response loss", + ))) + } + + fn strings(args: &[Vec]) -> Vec { + args.iter() + .map(|arg| String::from_utf8_lossy(arg).into_owned()) + .collect() + } + + fn block_on(future: impl std::future::Future) -> T { + futures::executor::block_on(future) + } + + fn tracked(value_key: &str, watermark_key: &str) -> ReadRequest { + ReadRequest { + value_key: value_key.to_string(), + watermark_key: Some(watermark_key.to_string()), + } + } + + fn context() -> ReadContext { + ReadContext { + timeout_ms: 50, + cancel: CancelToken::new(), + } + } + + fn go_invalidation_script() -> String { + let path = Path::new(env!("CARGO_MANIFEST_DIR")).join("../go/redis_adapter.go"); + let source = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("read {}: {error}", path.display())); + let start = source + .find("const InvalidationScript = `") + .expect("Go declares InvalidationScript") + + "const InvalidationScript = `".len(); + let end = source[start..] + .find('`') + .expect("Go raw string literal closes"); + source[start..start + end].to_string() + } + + #[test] + fn invalidation_script_matches_go_byte_for_byte() { + assert_eq!(INVALIDATION_SCRIPT, go_invalidation_script()); + } + + #[test] + fn invalidation_script_sha1_matches_independent_digests() { + let sha = invalidation_script_sha1(); + // Pinned from `shasum -a 1` over the Go constant. TypeScript's + // INVALIDATE_CACHE_SCRIPT_SHA1 (e90603d37a1a3d7a49ae5417bea0d595b94ddec6) + // hashes the same Lua with different formatting. + assert_eq!(sha, "a6c1c661884bd7f535a13c79135a40b1edd2e216"); + assert_eq!(sha, redis::Script::new(INVALIDATION_SCRIPT).get_hash()); + } + + #[test] + fn write_is_one_set_with_px_and_validates_ok() { + let frame = Frame { + created_at_ms: 1_700_000_000_000, + payload: Payload::binary(vec![0, 1, 255]), + }; + let connection = Scripted::with(vec![Ok(Value::Okay)]); + let adapter = RedisAdapter::new(connection.clone()); + block_on(adapter.write(WriteRequest { + value_key: "value".into(), + frame: frame.clone(), + ttl_ms: 2, + })) + .expect("SET succeeds"); + let calls = connection.calls(); + assert_eq!(calls.len(), 1); + assert!(!calls[0].primary); + assert_eq!( + calls[0].args, + vec![ + b"SET".to_vec(), + b"value".to_vec(), + encode_frame(&frame).unwrap(), + b"PX".to_vec(), + b"2".to_vec(), + ] + ); + + for ttl_ms in [0, MAX_SUPPORTED_DURATION_MS + 1] { + let error = block_on(adapter.write(WriteRequest { + value_key: "value".into(), + frame: frame.clone(), + ttl_ms, + })) + .expect_err("invalid TTL rejected"); + assert!(error.is::(), "{error}"); + } + assert_eq!(connection.calls().len(), 1, "invalid input dispatched"); + + for reply in [ + Value::Nil, + Value::SimpleString("ok".into()), + Value::Boolean(true), + Value::Int(1), + ] { + assert!(validate_set_reply(&reply).is_err(), "{reply:?}"); + } + for reply in [ + Value::Okay, + Value::SimpleString("OK".into()), + Value::BulkString(b"OK".to_vec()), + ] { + assert!(validate_set_reply(&reply).is_ok(), "{reply:?}"); + } + } + + #[test] + fn write_reports_protocol_error_on_bad_reply() { + let connection = Scripted::with(vec![Ok(Value::Int(1))]); + let adapter = RedisAdapter::new(connection); + let error = block_on(adapter.write(WriteRequest { + value_key: "value".into(), + frame: Frame { + created_at_ms: 1, + payload: Payload::text("1"), + }, + ttl_ms: 1, + })) + .expect_err("bad reply rejected"); + assert!(error.is::(), "{error}"); + } + + #[test] + fn invalidate_retries_evalsha_once_as_eval_with_identical_arguments() { + let connection = Scripted::with(vec![failure(), Ok(Value::Int(1))]); + let adapter = RedisAdapter::new(connection.clone()); + block_on(adapter.invalidate(InvalidateRequest { + watermark_key: "watermark".into(), + invalidated_at_ms: 1_700_000_000_000, + future_buffer_ms: 100, + })) + .expect("EVAL retry succeeds"); + let calls = connection.calls(); + assert_eq!(calls.len(), 2, "{calls:?}"); + let first = strings(&calls[0].args); + let second = strings(&calls[1].args); + assert_eq!(first[0], "EVALSHA"); + assert_eq!(first[1], invalidation_script_sha1()); + assert_eq!(second[0], "EVAL"); + assert_eq!(second[1], INVALIDATION_SCRIPT); + assert_eq!(first[2..], second[2..], "retry changed logical arguments"); + assert_eq!( + first[2..], + ["1", "watermark", "100", "1700000000000"].map(str::to_string) + ); + } + + #[test] + fn invalidate_failed_retry_surfaces_and_stops() { + let connection = Scripted::with(vec![failure(), failure()]); + let adapter = RedisAdapter::new(connection.clone()); + let error = block_on(adapter.invalidate_decimal("watermark", "0", "1")) + .expect_err("second failure surfaces"); + assert!(error.is::(), "{error}"); + assert_eq!(connection.calls().len(), 2); + } + + #[test] + fn invalidate_accepted_invalid_reply_is_not_retried() { + let connection = Scripted::with(vec![Ok(Value::BulkString(b"1".to_vec()))]); + let adapter = RedisAdapter::new(connection.clone()); + let error = block_on(adapter.invalidate(InvalidateRequest { + watermark_key: "watermark".into(), + invalidated_at_ms: 1, + future_buffer_ms: 0, + })) + .expect_err("string reply rejected"); + assert!(error.is::(), "{error}"); + assert_eq!( + connection.calls().len(), + 1, + "invalid accepted reply retried" + ); + + for reply in [ + Value::Nil, + Value::BulkString(Vec::new()), + Value::Int(0), + Value::Double(1.0), + Value::BulkString(b"1".to_vec()), + Value::SimpleString("1".into()), + Value::Boolean(true), + Value::Okay, + ] { + assert!(validate_invalidation_reply(&reply).is_err(), "{reply:?}"); + } + assert!(validate_invalidation_reply(&Value::Int(1)).is_ok()); + } + + #[test] + fn invalidate_rejects_out_of_domain_arguments_before_dispatch() { + let connection = Scripted::with(vec![]); + let adapter = RedisAdapter::new(connection.clone()); + for (invalidated_at_ms, future_buffer_ms) in [ + (0, MAX_SUPPORTED_DURATION_MS + 1), + (MAX_SAFE_INTEGER, 1), + (MAX_SAFE_INTEGER + 1, 0), + ] { + let error = block_on(adapter.invalidate(InvalidateRequest { + watermark_key: "watermark".into(), + invalidated_at_ms, + future_buffer_ms, + })) + .expect_err("invalid domain rejected"); + assert!(error.is::(), "{error}"); + } + assert!(connection.calls().is_empty()); + } + + #[test] + fn tracked_read_is_one_primary_mget_classified_by_the_protocol() { + let raw = encode_frame(&Frame { + created_at_ms: 2, + payload: Payload::text("1"), + }) + .unwrap(); + let bulk = |bytes: &[u8]| Value::BulkString(bytes.to_vec()); + struct Case { + reply: Value, + expected: Result, + } + let hit = || { + Ok(ReadResult::Hit(Frame { + created_at_ms: 2, + payload: Payload::text("1"), + })) + }; + let miss = |reason, observed| { + Ok(ReadResult::Miss { + reason, + observed_watermark_ms: observed, + }) + }; + let cases = [ + Case { + reply: Value::Array(vec![bulk(&raw), bulk(b"1")]), + expected: hit(), + }, + Case { + reply: Value::Array(vec![bulk(&raw), bulk(b"2")]), + expected: miss(MissReason::WatermarkFenced, Some(2)), + }, + Case { + reply: Value::Array(vec![Value::Nil, bulk(b"5")]), + expected: miss(MissReason::ValueAbsent, Some(5)), + }, + Case { + reply: Value::Array(vec![bulk(&raw), Value::Nil]), + expected: hit(), + }, + Case { + reply: Value::Array(vec![bulk(&raw), bulk(b"bad")]), + expected: miss(MissReason::Unclassified, None), + }, + Case { + reply: Value::Array(vec![bulk(&raw)]), + expected: Err(()), + }, + Case { + reply: Value::Array(vec![Value::Int(12), Value::Nil]), + expected: Err(()), + }, + Case { + reply: Value::Array(vec![bulk(&raw), Value::Boolean(false)]), + expected: Err(()), + }, + Case { + reply: bulk(&raw), + expected: Err(()), + }, + ]; + for case in cases { + let connection = Scripted::with(vec![Ok(case.reply.clone())]); + let adapter = RedisAdapter::new(connection.clone()); + let got = block_on(adapter.read(tracked("value", "watermark"), context())); + match (&case.expected, &got) { + (Ok(expected), Ok(got)) => assert_eq!(got, expected, "{:?}", case.reply), + (Err(()), Err(error)) => { + assert!( + error.is::(), + "{:?}: {error}", + case.reply + ) + } + _ => panic!("reply {:?}: {got:?}", case.reply), + } + assert_eq!( + connection.calls(), + vec![Call { + primary: true, + args: vec![b"MGET".to_vec(), b"value".to_vec(), b"watermark".to_vec()], + }] + ); + } + } + + #[test] + fn untracked_read_is_one_get() { + let raw = encode_frame(&Frame { + created_at_ms: 7, + payload: Payload::binary(vec![0, 255]), + }) + .unwrap(); + let connection = Scripted::with(vec![Ok(Value::Nil), Ok(Value::BulkString(raw))]); + let adapter = RedisAdapter::new(connection.clone()); + let request = ReadRequest { + value_key: "value".into(), + watermark_key: None, + }; + assert_eq!( + block_on(adapter.read(request.clone(), context())).unwrap(), + ReadResult::miss(MissReason::ValueAbsent) + ); + assert_eq!( + block_on(adapter.read(request, context())).unwrap(), + ReadResult::Hit(Frame { + created_at_ms: 7, + payload: Payload::binary(vec![0, 255]), + }) + ); + let calls = connection.calls(); + assert_eq!(calls.len(), 2); + for call in calls { + assert_eq!( + call, + Call { + primary: false, + args: vec![b"GET".to_vec(), b"value".to_vec()], + } + ); + } + } + + #[test] + fn read_payload_encoding_error_is_an_error_not_a_miss() { + let mut raw = encode_frame(&Frame { + created_at_ms: 2, + payload: Payload::text("1"), + }) + .unwrap(); + raw[9] = 7; + let connection = Scripted::with(vec![Ok(Value::BulkString(raw))]); + let adapter = RedisAdapter::new(connection); + let error = block_on(adapter.read( + ReadRequest { + value_key: "value".into(), + watermark_key: None, + }, + context(), + )) + .expect_err("unknown encoding tag is an error"); + assert_eq!( + error.downcast_ref::(), + Some(&ProtocolError::PayloadEncoding) + ); + } + + #[test] + fn read_honors_cancellation_before_and_during_the_wait() { + let connection = Scripted::with(vec![]); + let adapter = RedisAdapter::new(connection.clone()); + let cancelled = ReadContext { + timeout_ms: 50, + cancel: CancelToken::new(), + }; + cancelled.cancel.cancel(); + let error = block_on(adapter.read(tracked("value", "watermark"), cancelled)) + .expect_err("pre-cancelled read never dispatches"); + assert!(error.is::(), "{error}"); + assert!(connection.calls().is_empty()); + + let context = context(); + let token = context.cancel.clone(); + let canceller = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(20)); + token.cancel(); + }); + let error = block_on(adapter.read(tracked("value", "watermark"), context)) + .expect_err("hung command is abandoned on cancel"); + canceller.join().unwrap(); + assert!(error.is::(), "{error}"); + assert_eq!( + connection.calls().len(), + 1, + "the command was dispatched once" + ); + } +} diff --git a/rust/tests/formal/invalidation_vectors.rs b/rust/tests/formal/invalidation_vectors.rs new file mode 100644 index 00000000..08faadf0 --- /dev/null +++ b/rust/tests/formal/invalidation_vectors.rs @@ -0,0 +1,327 @@ +//! Real-server replay of the invalidation transition vectors +//! (`formal/PROTOCOL.md`, "Invalidation vector schema 2"). +//! +//! Mirrors `go/redis_integration_test.go` `testInvalidationVectors`: the 49 +//! fixed vectors of `formal/invalidation-vectors.json` are merged with the +//! 288 Quint-generated vectors of `formal/quint-invalidation-vectors.json` +//! after the generated corpus's provenance fingerprints are verified. Each +//! vector installs `existing` atomically, runs the adapter's raw decimal +//! invalidation, then observes type, content and retention atomically and +//! compares them with `expected.state`. TTL comparison subtracts only the +//! server time measured between setup and observation; persistence (`-1`) +//! and absence (`-2`) are exact. +//! +//! Every function takes the adapter and a connection so the including test +//! binary owns Docker orchestration and stays small. Include with +//! `#[path = "formal/invalidation_vectors.rs"] mod invalidation_vectors;`. + +#![allow(dead_code)] + +use std::fs; +use std::path::Path; + +use dialcache::redis::{RedisAdapter, RedisConnection}; +use redis::Value; +use serde::Deserialize; + +/// Fixed corpus size pinned by every port. +pub const FIXED_VECTORS: usize = 49; +/// Generated corpus size pinned by every port. +pub const GENERATED_VECTORS: usize = 288; +const SCHEMA_VERSION: u64 = 2; +const MODEL: &str = "formal/dialcache-invalidation-transition.qnt"; +const GENERATOR: &str = "formal/generate-invalidation-vectors.mjs"; + +/// One tagged Redis key state: `absent`, `string` with `value`, or `list` +/// with ordered `values`; `ttl_ms` is `-2` absent, `-1` persistent or positive. +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct State { + pub kind: String, + #[serde(default)] + pub value: Option, + #[serde(default)] + pub values: Option>, + pub ttl_ms: i64, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Expected { + #[serde(default)] + pub error: bool, + pub state: State, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct Vector { + pub name: String, + pub existing: State, + /// Raw decimal argument text, passed to the script unparsed. + pub future_buffer_ms: String, + /// Raw decimal argument text, passed to the script unparsed. + pub invalidated_at_ms: String, + pub expected: Expected, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Corpus { + schema_version: u64, + #[serde(default)] + provenance: Option, + vectors: Vec, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct Provenance { + model: String, + source_sha256: std::collections::BTreeMap, +} + +fn read(repo_root: &Path, relative: &str) -> Vec { + let path = repo_root.join(relative); + fs::read(&path).unwrap_or_else(|error| panic!("read {}: {error}", path.display())) +} + +/// Load and merge both corpora exactly as Go does. `sha256_hex` fingerprints +/// the generator's model and script sources so stale generated vectors are +/// rejected rather than replayed. +pub fn load_corpus(repo_root: &Path, sha256_hex: impl Fn(&[u8]) -> String) -> Vec { + let fixed: Corpus = + serde_json::from_slice(&read(repo_root, "formal/invalidation-vectors.json")) + .expect("parse fixed invalidation corpus"); + assert_eq!( + fixed.schema_version, SCHEMA_VERSION, + "unsupported fixed corpus schema" + ); + assert_eq!( + fixed.vectors.len(), + FIXED_VECTORS, + "unsupported fixed corpus size" + ); + + let generated: Corpus = + serde_json::from_slice(&read(repo_root, "formal/quint-invalidation-vectors.json")) + .expect("parse Quint invalidation corpus"); + assert_eq!( + generated.schema_version, SCHEMA_VERSION, + "unsupported Quint corpus schema" + ); + assert_eq!( + generated.vectors.len(), + GENERATED_VECTORS, + "unsupported Quint corpus size" + ); + let provenance = generated + .provenance + .expect("Quint corpus records provenance"); + assert_eq!( + provenance.model, MODEL, + "invalid Quint invalidation provenance" + ); + assert_eq!( + provenance.source_sha256.len(), + 2, + "invalid Quint invalidation provenance" + ); + for source in [MODEL, GENERATOR] { + let actual = sha256_hex(&read(repo_root, source)); + assert_eq!( + provenance.source_sha256.get(source), + Some(&actual), + "stale Quint invalidation vectors for {source}; regenerate and review" + ); + } + for (index, vector) in generated.vectors.iter().enumerate() { + assert!( + vector.name.starts_with(&format!("Quint {index:03}: ")), + "incomplete or reordered Quint invalidation combinations at {index}: {}", + vector.name + ); + } + + let mut vectors = fixed.vectors; + vectors.extend(generated.vectors); + vectors +} + +/// Install `existing` atomically and return the server clock in epoch ms. +const SETUP: &str = r#"redis.replicate_commands() +local now=redis.call("TIME") +redis.call("DEL",KEYS[1]) +if ARGV[1]=="string" then redis.call("SET",KEYS[1],ARGV[2]) end +if ARGV[1]=="list" then for _,value in ipairs(cjson.decode(ARGV[2])) do redis.call("RPUSH",KEYS[1],value) end end +if tonumber(ARGV[3])>0 then redis.call("PEXPIRE",KEYS[1],ARGV[3]) end +return tonumber(now[1])*1000+math.floor(tonumber(now[2])/1000)"#; + +/// Observe type, content, retention and the server clock atomically. +const OBSERVE: &str = r#"local kind=redis.call("TYPE",KEYS[1]).ok +local content={} +if kind=="string" then content=redis.call("GET",KEYS[1]) end +if kind=="list" then content=redis.call("LRANGE",KEYS[1],0,-1) end +if kind=="none" then kind="absent" end +local ttl=redis.call("PTTL",KEYS[1]);local now=redis.call("TIME") +return {kind,content,ttl,tonumber(now[1])*1000+math.floor(tonumber(now[2])/1000)}"#; + +fn text(value: &Value) -> Result { + match value { + Value::BulkString(bytes) => { + String::from_utf8(bytes.clone()).map_err(|error| format!("non-UTF-8 reply: {error}")) + } + Value::SimpleString(text) => Ok(text.clone()), + other => Err(format!("expected a string reply, got {other:?}")), + } +} + +fn integer(value: &Value) -> Result { + match value { + Value::Int(integer) => Ok(*integer), + other => Err(format!("expected an integer reply, got {other:?}")), + } +} + +/// What the server holds after the transition. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Observed { + pub kind: String, + pub value: Option, + pub values: Option>, + pub ttl_ms: i64, + pub now_ms: i64, +} + +fn observed(reply: Value) -> Result { + let Value::Array(items) = reply else { + return Err(format!("observe returned {reply:?}")); + }; + let [kind, content, ttl, now]: [Value; 4] = items + .try_into() + .map_err(|items| format!("observe returned {items:?}"))?; + let kind = text(&kind)?; + let (value, values) = match kind.as_str() { + "string" => (Some(text(&content)?), None), + "list" => { + let Value::Array(entries) = content else { + return Err(format!("list content {content:?}")); + }; + let entries = entries.iter().map(text).collect::, _>>()?; + (None, Some(entries)) + } + _ => (None, None), + }; + Ok(Observed { + kind, + value, + values, + ttl_ms: integer(&ttl)?, + now_ms: integer(&now)?, + }) +} + +/// Compare an observed state with `want` under the PROTOCOL.md TTL rule. +pub fn compare(want: &State, got: &Observed, elapsed_ms: i64) -> Result<(), String> { + if elapsed_ms < 0 { + return Err("server time moved backwards".to_string()); + } + if got.kind != want.kind { + return Err(format!("kind {} want {}", got.kind, want.kind)); + } + match want.kind.as_str() { + "string" if got.value != want.value => { + return Err(format!("content {:?} want {:?}", got.value, want.value)); + } + "list" if got.values != want.values => { + return Err(format!("list {:?} want {:?}", got.values, want.values)); + } + _ => {} + } + if want.ttl_ms < 0 { + if got.ttl_ms != want.ttl_ms { + return Err(format!("TTL {} want {}", got.ttl_ms, want.ttl_ms)); + } + } else { + let minimum = (want.ttl_ms - elapsed_ms).max(0); + if got.ttl_ms < minimum || got.ttl_ms > want.ttl_ms { + return Err(format!( + "TTL {} outside [{minimum},{}], measured elapsed={elapsed_ms}", + got.ttl_ms, want.ttl_ms + )); + } + } + Ok(()) +} + +/// Replay one vector against `key` through `adapter`; fixture commands run on +/// the key's primary through `connection`. +pub async fn replay( + adapter: &RedisAdapter, + connection: &C, + key: &str, + vector: &Vector, +) -> Result<(), String> { + let content = match vector.existing.kind.as_str() { + "list" => serde_json::to_string(vector.existing.values.as_deref().unwrap_or_default()) + .map_err(|error| error.to_string())?, + _ => vector.existing.value.clone().unwrap_or_default(), + }; + let mut setup = redis::cmd("EVAL"); + setup + .arg(SETUP) + .arg(1) + .arg(key) + .arg(&vector.existing.kind) + .arg(content) + .arg(vector.existing.ttl_ms); + let start_ms = integer( + &connection + .run_on_primary(key, setup) + .await + .map_err(|error| format!("setup: {error}"))?, + )?; + + let result = adapter + .invalidate_decimal(key, &vector.future_buffer_ms, &vector.invalidated_at_ms) + .await; + if result.is_err() != vector.expected.error { + return Err(format!( + "error={:?} expected rejection={}", + result.err().map(|error| error.to_string()), + vector.expected.error + )); + } + + let mut observe = redis::cmd("EVAL"); + observe.arg(OBSERVE).arg(1).arg(key); + let got = observed( + connection + .run_on_primary(key, observe) + .await + .map_err(|error| format!("observe: {error}"))?, + )?; + compare(&vector.expected.state, &got, got.now_ms - start_ms) +} + +/// Replay every vector under `key_prefix`. Returns the number replayed, or +/// every failure as `name: reason`. +pub async fn replay_all( + adapter: &RedisAdapter, + connection: &C, + key_prefix: &str, + vectors: &[Vector], +) -> Result> { + let mut failures = Vec::new(); + for vector in vectors { + let key = format!("{key_prefix}{}", vector.name); + if let Err(reason) = replay(adapter, connection, &key, vector).await { + failures.push(format!("{}: {reason}", vector.name)); + } + } + if failures.is_empty() { + Ok(vectors.len()) + } else { + Err(failures) + } +} diff --git a/rust/tests/redis_integration.rs b/rust/tests/redis_integration.rs new file mode 100644 index 00000000..d7ca251f --- /dev/null +++ b/rust/tests/redis_integration.rs @@ -0,0 +1,486 @@ +//! Real-server tests of the `redis` adapter, mirroring +//! `go/redis_integration_test.go`: Docker-started Redis 6.2, Redis 7 and +//! Valkey 8 standalone servers plus a single-node Redis 7 cluster, each +//! exercised for complete-frame round trips, watermark fencing, the +//! `EVALSHA` to `EVAL` recovery after `SCRIPT FLUSH`, and the full +//! invalidation vector corpus (`formal/invalidation_vectors.rs`). +//! +//! Every test returns early with a printed message unless +//! `DIALCACHE_RUST_INTEGRATION=1` is set; with it set, the `docker` CLI and +//! the images must be available. Containers are removed by a drop guard on +//! every exit path. +//! +//! Cluster coverage is a single primary with every slot assigned: the +//! adapter's explicit primary routing is exercised through a real +//! `ClusterConnection`, but with no replicas present the test cannot +//! distinguish primary from replica routing the way the Go six-node test +//! does (the `redis` crate exposes no dialer hook to remap the internal +//! container addresses a multi-node cluster announces). + +#![cfg(feature = "redis")] + +mod formal; +#[path = "formal/invalidation_vectors.rs"] +mod invalidation_vectors; + +use std::net::TcpListener; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +use dialcache::protocol::encode_frame; +use dialcache::redis::{RedisAdapter, RedisConnection}; +use dialcache::{ + CancelToken, Frame, InvalidateRequest, MissReason, Payload, ReadContext, ReadRequest, + ReadResult, Remote, WriteRequest, +}; +use formal::witness::sha256_hex; +use redis::aio::{ConnectionManager, ConnectionManagerConfig}; +use redis::cluster::ClusterClientBuilder; +use redis::cluster_async::ClusterConnection; +use redis::{Client, Value}; + +const ENABLE_ENV: &str = "DIALCACHE_RUST_INTEGRATION"; +const COMMAND_BUDGET: Duration = Duration::from_secs(2); +const READY_BUDGET: Duration = Duration::from_secs(15); + +fn enabled(test: &str) -> bool { + if std::env::var(ENABLE_ENV).as_deref() == Ok("1") { + return true; + } + println!("skipping {test}: set {ENABLE_ENV}=1 to run Docker-backed Redis tests"); + false +} + +fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("..") +} + +/// Run one `docker` command and return its trimmed stdout. A cold `docker +/// run` writes pull progress to stderr and only the container id to stdout. +fn docker(args: &[&str]) -> Result { + let output = Command::new("docker") + .args(args) + .output() + .map_err(|error| format!("docker {args:?}: {error}"))?; + if !output.status.success() { + return Err(format!( + "docker {args:?}: {}\nstdout:\n{}\nstderr:\n{}", + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )); + } + Ok(String::from_utf8_lossy(&output.stdout).trim().to_string()) +} + +/// A started container, force-removed when dropped. +struct Container { + id: String, +} + +impl Drop for Container { + fn drop(&mut self) { + let _ = Command::new("docker").args(["rm", "-f", &self.id]).output(); + } +} + +/// A free loopback port; the listener is dropped so Docker can bind it. +fn free_port() -> u16 { + TcpListener::bind("127.0.0.1:0") + .and_then(|listener| listener.local_addr()) + .expect("bind an ephemeral loopback port") + .port() +} + +/// `docker run` a Redis-compatible image with the Go test's server flags. +/// For a cluster node the host port is chosen first so the node can announce +/// the loopback address a host-side cluster client can reach. +fn start_server(image: &str, cluster: bool) -> (Container, String) { + let server = if image.contains("valkey") { + "valkey-server" + } else { + "redis-server" + }; + let announce_port = cluster.then(free_port); + let publish = match announce_port { + Some(port) => format!("127.0.0.1:{port}:6379"), + None => "127.0.0.1::6379".to_string(), + }; + let announce = announce_port.map(|port| port.to_string()); + let mut args = vec![ + "run", + "-d", + "--rm", + "-p", + &publish, + image, + server, + "--save", + "", + "--appendonly", + "no", + "--protected-mode", + "no", + ]; + if let Some(port) = &announce { + args.extend([ + "--cluster-enabled", + "yes", + "--cluster-config-file", + "/tmp/nodes.conf", + "--cluster-node-timeout", + "5000", + "--cluster-announce-ip", + "127.0.0.1", + "--cluster-announce-port", + port, + ]); + } + let id = docker(&args).expect("start container"); + let container = Container { id }; + let endpoint = docker(&["port", &container.id, "6379/tcp"]) + .expect("published port") + .lines() + .next() + .expect("one published endpoint") + .to_string(); + (container, endpoint) +} + +async fn wait_for_ping(endpoint: &str) { + let client = Client::open(format!("redis://{endpoint}")).expect("client url"); + let deadline = Instant::now() + READY_BUDGET; + loop { + if let Ok(mut connection) = client.get_multiplexed_async_connection().await { + if redis::cmd("PING") + .query_async::(&mut connection) + .await + .is_ok() + { + return; + } + } + assert!(Instant::now() < deadline, "{endpoint} did not become ready"); + tokio::time::sleep(Duration::from_millis(25)).await; + } +} + +async fn standalone(image: &str) -> (Container, ConnectionManager) { + let (container, endpoint) = start_server(image, false); + wait_for_ping(&endpoint).await; + let client = Client::open(format!("redis://{endpoint}")).expect("client url"); + let config = ConnectionManagerConfig::new() + .set_connection_timeout(Some(COMMAND_BUDGET)) + .set_response_timeout(Some(COMMAND_BUDGET)) + .set_number_of_retries(1); + let connection = client + .get_connection_manager_with_config(config) + .await + .expect("connection manager"); + (container, connection) +} + +/// One cluster-enabled node owning every slot; readiness waits for the +/// node's own slot view exactly as the Go test waits for six nodes. +async fn single_node_cluster(image: &str) -> (Container, ClusterConnection) { + let (container, endpoint) = start_server(image, true); + wait_for_ping(&endpoint).await; + docker(&[ + "exec", + &container.id, + "redis-cli", + "CLUSTER", + "ADDSLOTSRANGE", + "0", + "16383", + ]) + .expect("assign every slot"); + let deadline = Instant::now() + READY_BUDGET; + loop { + let info = + docker(&["exec", &container.id, "redis-cli", "CLUSTER", "INFO"]).unwrap_or_default(); + let field = |name: &str| { + info.lines() + .find_map(|line| line.trim().strip_prefix(name)?.strip_prefix(':')) + .unwrap_or("") + .to_string() + }; + if field("cluster_state") == "ok" + && field("cluster_slots_assigned") == "16384" + && field("cluster_slots_ok") == "16384" + { + break; + } + assert!( + Instant::now() < deadline, + "cluster did not become ready:\n{info}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + let connection = ClusterClientBuilder::new([format!("redis://{endpoint}")]) + .connection_timeout(COMMAND_BUDGET) + .response_timeout(COMMAND_BUDGET) + .build() + .expect("cluster client") + .get_async_connection() + .await + .expect("cluster connection"); + (container, connection) +} + +fn context() -> ReadContext { + ReadContext { + timeout_ms: 50, + cancel: CancelToken::new(), + } +} + +fn tracked(value_key: &str, watermark_key: &str) -> ReadRequest { + ReadRequest { + value_key: value_key.to_string(), + watermark_key: Some(watermark_key.to_string()), + } +} + +async fn primary_integer(connection: &C, key: &str, cmd: redis::Cmd) -> i64 { + match connection + .run_on_primary(key, cmd) + .await + .expect("fixture command") + { + Value::Int(value) => value, + other => panic!("expected integer, got {other:?}"), + } +} + +/// `go/redis_integration_test.go` `testPrimaryRead` plus untracked reads: +/// stored bytes are the exact frame, a value write never extends the +/// watermark, the fence is observed on the primary, and `SCRIPT FLUSH` +/// forces the `EVALSHA` recovery through the real server. +async fn round_trip(adapter: &RedisAdapter, connection: &C, label: &str) { + let key = format!("{{primary-rust-{label}}}:value"); + let watermark = format!("{{primary-rust-{label}}}:watermark"); + let untracked = format!("{{primary-rust-{label}}}:untracked"); + + let frame = Frame { + created_at_ms: 2_000, + payload: Payload::binary(vec![0, 1, 255]), + }; + let text_frame = Frame { + created_at_ms: 3_000, + payload: Payload::text("DialCache é"), + }; + adapter + .write(WriteRequest { + value_key: untracked.clone(), + frame: text_frame.clone(), + ttl_ms: 10_000, + }) + .await + .expect("untracked write"); + let got = adapter + .read( + ReadRequest { + value_key: untracked.clone(), + watermark_key: None, + }, + context(), + ) + .await + .expect("untracked read"); + assert_eq!( + got, + ReadResult::Hit(text_frame), + "{label}: untracked round trip" + ); + let got = adapter + .read( + ReadRequest { + value_key: format!("{untracked}:missing"), + watermark_key: None, + }, + context(), + ) + .await + .expect("absent read"); + assert_eq!(got, ReadResult::miss(MissReason::ValueAbsent), "{label}"); + + // Tracked frame before any watermark: served. + adapter + .write(WriteRequest { + value_key: key.clone(), + frame: frame.clone(), + ttl_ms: 10_000, + }) + .await + .expect("tracked write"); + let got = adapter + .read(tracked(&key, &watermark), context()) + .await + .expect("tracked read"); + assert_eq!( + got, + ReadResult::Hit(frame.clone()), + "{label}: tracked round trip" + ); + + adapter + .invalidate(InvalidateRequest { + watermark_key: watermark.clone(), + invalidated_at_ms: 2_000, + future_buffer_ms: 0, + }) + .await + .expect("invalidate"); + let mut pttl = redis::cmd("PTTL"); + pttl.arg(&watermark); + let before = primary_integer(connection, &key, pttl.clone()).await; + adapter + .write(WriteRequest { + value_key: key.clone(), + frame: frame.clone(), + ttl_ms: 10_000, + }) + .await + .expect("tracked write"); + let mut get = redis::cmd("GET"); + get.arg(&key); + let raw = connection.run_on_primary(&key, get).await.expect("GET"); + assert_eq!( + raw, + Value::BulkString(encode_frame(&frame).unwrap()), + "{label}: stored frame differs" + ); + let got = adapter + .read(tracked(&key, &watermark), context()) + .await + .expect("tracked read"); + assert_eq!( + got, + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(2_000), + }, + "{label}: tracked read missed primary fence" + ); + let after = primary_integer(connection, &key, pttl).await; + assert!( + after <= before, + "{label}: value write extended watermark {before} -> {after}" + ); + + // A newer frame passes the fence. + let newer = Frame { + created_at_ms: 2_001, + payload: Payload::text("1"), + }; + adapter + .write(WriteRequest { + value_key: key.clone(), + frame: newer.clone(), + ttl_ms: 10_000, + }) + .await + .expect("newer write"); + let got = adapter + .read(tracked(&key, &watermark), context()) + .await + .expect("tracked read"); + assert_eq!(got, ReadResult::Hit(newer), "{label}: newer frame fenced"); + + // SCRIPT FLUSH forces the adapter's EVALSHA recovery through the real server. + let mut flush = redis::cmd("SCRIPT"); + flush.arg("FLUSH"); + connection + .run_on_primary(&key, flush) + .await + .expect("SCRIPT FLUSH"); + let mut evalsha = redis::cmd("EVALSHA"); + evalsha + .arg(dialcache::redis::invalidation_script_sha1()) + .arg(1) + .arg(&watermark) + .arg("0") + .arg("3000"); + let noscript = connection + .run_on_primary(&key, evalsha) + .await + .expect_err("flushed script cache rejects EVALSHA"); + assert_eq!(noscript.code(), Some("NOSCRIPT"), "{label}: {noscript}"); + adapter + .invalidate(InvalidateRequest { + watermark_key: watermark.clone(), + invalidated_at_ms: 3_000, + future_buffer_ms: 0, + }) + .await + .expect("NOSCRIPT recovery"); + let got = adapter + .read(tracked(&key, &watermark), context()) + .await + .expect("tracked read"); + assert_eq!( + got, + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(3_000), + }, + "{label}: recovered invalidation did not fence" + ); +} + +async fn exercise(connection: C, label: &str) { + let adapter = RedisAdapter::new(connection); + round_trip(&adapter, adapter.connection(), label).await; + let vectors = invalidation_vectors::load_corpus(&repo_root(), sha256_hex); + let prefix = format!("{{rust-invalidation-{label}}}:"); + match invalidation_vectors::replay_all(&adapter, adapter.connection(), &prefix, &vectors).await + { + Ok(count) => println!("{label}: replayed {count} invalidation vectors"), + Err(failures) => panic!( + "{label}: {} of {} invalidation vectors failed:\n{}", + failures.len(), + vectors.len(), + failures.join("\n") + ), + } +} + +fn run_standalone(test: &str, image: &str) { + if !enabled(test) { + return; + } + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); + runtime.block_on(async { + let (_container, connection) = standalone(image).await; + exercise(connection, test).await; + }); +} + +#[test] +fn redis_6_2_standalone() { + run_standalone("redis6.2", "redis:6.2-alpine"); +} + +#[test] +fn redis_7_standalone() { + run_standalone("redis7", "redis:7-alpine"); +} + +#[test] +fn valkey_8_standalone() { + run_standalone("valkey8", "valkey/valkey:8-alpine"); +} + +#[test] +fn redis_7_single_node_cluster() { + if !enabled("cluster") { + return; + } + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); + runtime.block_on(async { + let (_container, connection) = single_node_cluster("redis:7-alpine").await; + exercise(connection, "cluster").await; + }); +} From a58ac0736e327f57bda2175c8239579a3e049b38 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 01:34:48 -0700 Subject: [PATCH 17/31] docs(rust): document every public item and warn on missing docs --- rust/src/cancel.rs | 1 + rust/src/clock.rs | 2 + rust/src/codec.rs | 15 +++ rust/src/datadog.rs | 6 + rust/src/engine.rs | 16 +++ rust/src/error.rs | 7 ++ rust/src/identity.rs | 18 +++ rust/src/lib.rs | 1 + rust/src/local.rs | 5 + rust/src/metrics.rs | 21 ++++ rust/src/observe.rs | 218 +++++++++++++++++++++++++++++++++- rust/src/operation.rs | 13 ++ rust/src/policy.rs | 36 ++++++ rust/src/preview.rs | 2 + rust/src/prometheus.rs | 7 ++ rust/src/protocol/envelope.rs | 13 ++ rust/src/protocol/frame.rs | 8 ++ rust/src/redis.rs | 3 + rust/src/remote.rs | 27 +++++ rust/src/testing.rs | 11 ++ rust/src/use_case.rs | 7 ++ 21 files changed, 432 insertions(+), 5 deletions(-) diff --git a/rust/src/cancel.rs b/rust/src/cancel.rs index f53f7795..736bac25 100644 --- a/rust/src/cancel.rs +++ b/rust/src/cancel.rs @@ -32,6 +32,7 @@ impl std::fmt::Debug for CancelToken { } impl CancelToken { + /// A token that has not been cancelled; clones share its state. pub fn new() -> Self { Self::default() } diff --git a/rust/src/clock.rs b/rust/src/clock.rs index b99bf871..ed7c85c6 100644 --- a/rust/src/clock.rs +++ b/rust/src/clock.rs @@ -38,6 +38,8 @@ pub struct SystemClock { } impl SystemClock { + /// A clock whose elapsed origin snaps to the process-wide millisecond + /// grid, so every default instance shares one local expiry grid. pub fn new() -> Self { let process = *PROCESS_ORIGIN.get_or_init(Instant::now); let now = Instant::now(); diff --git a/rust/src/codec.rs b/rust/src/codec.rs index 95875174..b5080939 100644 --- a/rust/src/codec.rs +++ b/rust/src/codec.rs @@ -14,12 +14,15 @@ use crate::error::BoxError; /// their exact bytes. Compression envelopes wrap either form transparently. #[derive(Debug, Clone, PartialEq, Eq, Hash, Default)] pub struct Payload { + /// The serializer output. Read as UTF-8 unless `binary`; ill-formed text + /// sequences are replaced when the payload is stored. pub bytes: Vec, /// `true` for binary output, `false` for UTF-8 text. pub binary: bool, } impl Payload { + /// A UTF-8 text payload. pub fn text(text: impl Into) -> Self { Payload { bytes: text.into().into_bytes(), @@ -27,6 +30,7 @@ impl Payload { } } + /// An opaque binary payload, stored byte-exact. pub fn binary(bytes: impl Into>) -> Self { Payload { bytes: bytes.into(), @@ -34,10 +38,12 @@ impl Payload { } } + /// Length in bytes. pub fn len(&self) -> usize { self.bytes.len() } + /// Whether the payload holds no bytes. pub fn is_empty(&self) -> bool { self.bytes.is_empty() } @@ -55,13 +61,19 @@ impl Payload { /// [`FromSync`]. Every decode must return an independent value: the cache may /// retain one payload and decode it more than once. pub trait Codec: Send + Sync + 'static { + /// Serialize `value`. A failure counts as a `serialization_dump` error + /// and skips the remote write. fn encode<'a>(&'a self, value: &'a T) -> BoxFuture<'a, Result>; + /// Deserialize `payload` into an independent value. A failure counts as + /// a `serialization_load` error and is treated as a miss. fn decode(&self, payload: Payload) -> BoxFuture<'_, Result>; } /// A synchronous codec. Wrap it in [`FromSync`] where a [`Codec`] is expected. pub trait SyncCodec: Send + Sync + 'static { + /// Serialize `value` without awaiting. fn encode(&self, value: &T) -> Result; + /// Deserialize `payload` into an independent value without awaiting. fn decode(&self, payload: Payload) -> Result; } @@ -94,10 +106,13 @@ pub const JSON_UNDEFINED_SENTINEL: &str = "__dialcache_json_undefined_v1__"; pub struct JsonCodec; impl JsonCodec { + /// Serialize any `Serialize` value as compact JSON text. pub fn encode_value(value: &T) -> Result { Ok(Payload::text(serde_json::to_string(value)?)) } + /// Deserialize JSON text, reading the TypeScript `undefined` sentinel as + /// JSON `null`. pub fn decode_value(payload: &Payload) -> Result { let text = payload.as_text(); if text == JSON_UNDEFINED_SENTINEL { diff --git a/rust/src/datadog.rs b/rust/src/datadog.rs index ae86dd38..60f49044 100644 --- a/rust/src/datadog.rs +++ b/rust/src/datadog.rs @@ -35,6 +35,7 @@ pub enum ObservationMetricType { /// Construction options of a [`DatadogObserver`]. #[derive(Debug, Clone, PartialEq, Eq)] pub struct DatadogOptions { + /// Whether observations go to DogStatsD histograms or distributions. pub observation_metric_type: ObservationMetricType, /// Metric-name namespace; unrelated to the cache namespace label. /// `None` selects [`DEFAULT_NAMESPACE`]; an explicit empty string is an error. @@ -42,6 +43,7 @@ pub struct DatadogOptions { } impl DatadogOptions { + /// Options under the default metric-name namespace, [`DEFAULT_NAMESPACE`]. pub fn new(observation_metric_type: ObservationMetricType) -> Self { DatadogOptions { observation_metric_type, @@ -49,6 +51,7 @@ impl DatadogOptions { } } + /// Set the metric-name namespace; it must satisfy [`is_valid_namespace`]. pub fn namespace(mut self, namespace: impl Into) -> Self { self.namespace = Some(namespace.into()); self @@ -152,6 +155,9 @@ impl fmt::Debug for DatadogObserver { } impl DatadogObserver { + /// Validate the namespace and precompute every metric name. Fails when + /// the namespace is invalid or a name would exceed + /// [`METRIC_NAME_MAX_LENGTH`]. pub fn new( client: impl DogStatsdClient, options: DatadogOptions, diff --git a/rust/src/engine.rs b/rust/src/engine.rs index 6a7b8069..beb17f75 100644 --- a/rust/src/engine.rs +++ b/rust/src/engine.rs @@ -124,14 +124,19 @@ impl Core { /// Exact process-scoped single-flight state of one instance. #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct ProcessCoalescingState { + /// Executions currently in flight on this instance, one per logical key. pub active_leaders: usize, + /// Callers currently waiting on those executions instead of running their own. pub active_followers: usize, + /// Milliseconds since the oldest in-flight execution started; `None` + /// when nothing is in flight. pub oldest_leader_age_ms: Option, } /// A point-in-time snapshot of cache-owned coalescing state. #[derive(Debug, Clone, PartialEq, Eq, Default)] pub struct CoalescingState { + /// Instance-wide single-flight state; request-scoped flights are not counted. pub process: ProcessCoalescingState, } @@ -210,11 +215,14 @@ impl DialCacheBuilder { self } + /// Receives structured log events. Defaults to the `log` crate facade + /// under target `dialcache`. pub fn logger(mut self, logger: impl Logger) -> Self { self.logger = Some(Arc::new(logger)); self } + /// Receives structured log events, shared. pub fn logger_arc(mut self, logger: Arc) -> Self { self.logger = Some(logger); self @@ -279,6 +287,8 @@ impl DialCacheBuilder { self } + /// Replace the wall and elapsed clock. Defaults to + /// [`SystemClock`](crate::SystemClock). pub fn clock(mut self, clock: impl Clock) -> Self { self.clock = Some(Arc::new(clock)); self @@ -302,6 +312,10 @@ impl DialCacheBuilder { self } + /// Validate the configuration and create the instance. Rejects a + /// namespace containing `{` or `}`, a read budget outside + /// `1..=2_147_483_647` ms, a zero shadow cap, an invalid compression + /// config, or a missing runtime when the `tokio` feature is off. pub fn build(self) -> Result { if self.namespace.contains(['{', '}']) { return Err(ConfigError::invalid( @@ -434,6 +448,8 @@ impl std::fmt::Debug for DialCache { } impl DialCache { + /// Start configuring an instance with the defaults documented on + /// [`DialCacheBuilder`]'s setters. pub fn builder() -> DialCacheBuilder { DialCacheBuilder::new() } diff --git a/rust/src/error.rs b/rust/src/error.rs index ed5cc60f..980411d4 100644 --- a/rust/src/error.rs +++ b/rust/src/error.rs @@ -88,7 +88,9 @@ impl From for Error { #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] #[error("DialCache fallback for use case {use_case:?} timed out after {timeout_ms} ms")] pub struct FallbackTimeout { + /// The use case whose source timed out. pub use_case: String, + /// The source budget that elapsed, in milliseconds. pub timeout_ms: u64, } @@ -96,17 +98,22 @@ pub struct FallbackTimeout { #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] #[error("DialCache remote read for use case {use_case:?} timed out after {timeout_ms} ms")] pub struct RemoteReadTimeout { + /// The use case whose remote read timed out. pub use_case: String, + /// The read budget that elapsed, in milliseconds. pub timeout_ms: u64, } /// Static configuration rejected at construction or registration. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum ConfigError { + /// A value is outside its domain; the message names it. #[error("{0}")] Invalid(String), + /// The use case name `watermark` is owned by invalidation. #[error("DialCache use case name is reserved: {0}")] ReservedUseCase(String), + /// A use case with this name is already registered on the instance. #[error("DialCache use case already registered: {0}")] UseCaseAlreadyRegistered(String), } diff --git a/rust/src/identity.rs b/rust/src/identity.rs index eb9546c0..743847ab 100644 --- a/rust/src/identity.rs +++ b/rust/src/identity.rs @@ -13,20 +13,29 @@ use crate::limits::FRAME_KEY_SUFFIX; /// use [`normalize_args`] to build them from a host-language record. #[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub struct Identity { + /// The instance namespace; empty means the cache fills in its own at execution. #[serde(default)] pub namespace: String, + /// The entity type; serialized as `keyType`. #[serde(rename = "keyType")] pub key_type: String, + /// The entity identifier. pub id: String, + /// The operation name; serialized as `useCase`. #[serde(rename = "useCase")] pub use_case: String, + /// Whether the identity is fenced by its entity's invalidation watermark; + /// serialized as `trackForInvalidation`. Defaults to false. #[serde(rename = "trackForInvalidation", default)] pub tracked: bool, + /// Secondary dimensions as `(name, spelled value)` pairs, already + /// normalized and in key order. #[serde(default)] pub args: Vec<(String, String)>, } impl Identity { + /// An untracked identity with an empty namespace and no arguments. pub fn new( key_type: impl Into, id: impl Into, @@ -42,16 +51,19 @@ impl Identity { } } + /// Set whether the identity shares its entity's invalidation watermark. pub fn tracked(mut self, tracked: bool) -> Self { self.tracked = tracked; self } + /// Set the namespace explicitly instead of inheriting the instance's. pub fn namespace(mut self, namespace: impl Into) -> Self { self.namespace = namespace.into(); self } + /// Replace the ordered argument pairs; build them with [`normalize_args`]. pub fn args(mut self, args: Vec<(String, String)>) -> Self { self.args = args; self @@ -77,8 +89,11 @@ pub struct Keys { /// Why an identity cannot form a key. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum IdentityError { + /// A component contains `{` or `}` where it would form the Redis Cluster + /// hash tag: the namespace always, the key type and ID when tracked. #[error("DialCache identity component contains a reserved hash-tag delimiter")] ReservedDelimiter, + /// An argument value cannot be spelled; the text names the offending value. #[error("DialCache identity contains an unsupported argument value: {0}")] UnsupportedArgument(String), } @@ -90,12 +105,15 @@ pub enum ArgValue { Absent, /// JSON null, spelled `null`. Null, + /// Spelled `true` or `false`. Bool(bool), /// Spelled with JavaScript `Number` formatting. Number(f64), + /// Spelled as a decimal integer. Int(i64), /// An arbitrary-precision integer as decimal text. BigInt(String), + /// Used verbatim, then percent-escaped like every other component. Str(String), } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 61b0c30f..7b3c392c 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -8,6 +8,7 @@ #![forbid(unsafe_code)] #![warn(missing_debug_implementations)] +#![warn(missing_docs)] pub mod cancel; pub mod clock; diff --git a/rust/src/local.rs b/rust/src/local.rs index 9b5ef017..39f3c871 100644 --- a/rust/src/local.rs +++ b/rust/src/local.rs @@ -12,9 +12,12 @@ pub type StoredValue = Arc; /// One process-local entry with its insertion-time TTL. #[derive(Clone)] pub struct LocalEntry { + /// The cached value, shared by reference with every reader. pub value: StoredValue, /// Whole-millisecond elapsed reading at insertion. pub inserted_ms: i64, + /// Lifetime in whole milliseconds; the entry is expired once + /// `now_ms - inserted_ms >= ttl_ms`. pub ttl_ms: i64, } @@ -45,10 +48,12 @@ impl LruLocalStore { } } + /// Entries currently held, expired ones included until a read removes them. pub fn len(&self) -> usize { self.entries.len() } + /// Whether no entries are held. pub fn is_empty(&self) -> bool { self.entries.is_empty() } diff --git a/rust/src/metrics.rs b/rust/src/metrics.rs index cf639148..1b1e8688 100644 --- a/rust/src/metrics.rs +++ b/rust/src/metrics.rs @@ -13,24 +13,45 @@ use crate::observe::Event; /// method names and the Go `metricKinds` table. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum MetricKind { + /// Counter of lookups that reached an enabled layer ([`Event::Request`]). Request, + /// Counter of classified misses ([`Event::Miss`]). Miss, + /// Counter of skipped layers and no-layer calls ([`Event::Disabled`]). Disabled, + /// Counter of bounded failure sites ([`Event::Error`]). Error, + /// Counter of explicit invalidation attempts ([`Event::Invalidation`]). Invalidation, + /// Counter of callers that joined an in-flight execution ([`Event::Coalesced`]). Coalesced, + /// Counter of terminal shadow job outcomes ([`Event::ShadowValidation`]). ShadowValidation, + /// Seconds: validated value age at a match or mismatch verdict ([`Event::ShadowValueAge`]). ShadowValueAge, + /// Seconds: how far a frame is dated ahead of the observing clock + /// ([`Event::FutureTimestampOffset`]). FutureTimestampOffset, + /// Counter of authorized stale-recovery checks ([`Event::StaleRecovery`]). StaleRecovery, + /// Seconds: age of a stale value when recovery served it + /// ([`Event::StaleRecoveryValueAge`]). StaleRecoveryValueAge, + /// Counter of envelope outcomes on writes and reads ([`Event::Compression`]). Compression, + /// Seconds: layer lookup latency ([`Event::Get`]). Get, + /// Seconds: time until the source settled or timed out ([`Event::Fallback`]). Fallback, + /// Seconds: codec encode or decode latency ([`Event::Serialization`]). Serialization, + /// Bytes: serializer output before compression and escaping ([`Event::Size`]). Size, + /// Bytes: prepared payload after compression and escaping ([`Event::StoredSize`]). StoredSize, + /// Ratio: stored over original bytes of a compressed write ([`Event::CompressionRatio`]). CompressionRatio, + /// Seconds: zstd compress or decompress latency ([`Event::CompressionDuration`]). CompressionDuration, } diff --git a/rust/src/observe.rs b/rust/src/observe.rs index 0140f42f..3a8ae64a 100644 --- a/rust/src/observe.rs +++ b/rust/src/observe.rs @@ -7,11 +7,13 @@ use crate::error::BoxError; pub use crate::remote::MissReason; macro_rules! str_enum { - ($(#[$meta:meta])* $name:ident { $($variant:ident => $text:literal),+ $(,)? }) => { + ($(#[$meta:meta])* $name:ident { $($(#[$vmeta:meta])* $variant:ident => $text:literal),+ $(,)? }) => { $(#[$meta])* #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] - pub enum $name { $($variant),+ } + pub enum $name { $($(#[$vmeta])* $variant),+ } impl $name { + /// The `snake_case` label value shared with the TypeScript and Go + /// ports and published by the bundled metric exporters. pub fn as_str(self) -> &'static str { match self { $($name::$variant => $text),+ } } @@ -25,27 +27,52 @@ macro_rules! str_enum { str_enum! { /// The cache layer a diagnostic refers to. Layer { + /// No layer was reached: caching was disabled for the scope, key + /// construction failed, or the policy could not be resolved. Noop => "noop", + /// The outermost enabled scope's request memo. RequestLocal => "request_local", + /// The process-local store. Local => "local", + /// The remote adapter on the caller-serving path. Remote => "remote", + /// The remote adapter inside detached shadow work: its reads, fills, + /// serialization, compression and payload sizes. RemoteShadow => "remote_shadow", } } str_enum! { /// Where an in-flight execution was shared. - CoalescingScope { RequestLocal => "request_local", Process => "process" } + CoalescingScope { + /// The flight was registered on the outermost enabled scope of the + /// same request. + RequestLocal => "request_local", + /// The flight was registered on the cache instance; separate + /// instances in one process do not share flights. + Process => "process", + } } str_enum! { /// Why a cache layer was skipped. DisabledReason { + /// Caching is not enabled for the call's scope; the source runs directly. Context => "context", + /// The layer has no effective TTL after the runtime overlay, which + /// includes the default of an omitted policy. PolicyDisabled => "policy_disabled", + /// The merged TTL leaf is not a whole number of seconds within 365 + /// days; only this layer is skipped, with a `config_resolution` error. InvalidTtl => "invalid_ttl", + /// The merged ramp leaf is not a finite number in `[0, 100]`; only + /// this layer is skipped, with a `config_resolution` error. InvalidRamp => "invalid_ramp", + /// The layer is configured but this key's stable cohort sample falls + /// outside its ramp. RampedDown => "ramped_down", + /// The whole invocation's policy could not be resolved (`noop` + /// layer), or the process-local read failed (`local` layer). ConfigError => "config_error", } } @@ -53,16 +80,34 @@ str_enum! { str_enum! { /// Stable failure sites. ErrorKind { + /// The key selector failed or panicked, or the identity could not + /// form a key. KeyConstruction => "key_construction", + /// The policy provider or resolution failed for the invocation, a + /// layer leaf was invalid, or a recovery or shadow option was rejected. ConfigResolution => "config_resolution", + /// A process-local read failed, or a remote read failed for a reason + /// other than its deadline. CacheRead => "cache_read", + /// A remote read did not answer within its read budget. CacheReadTimeout => "cache_read_timeout", + /// A process-local or remote write failed, or the write stamp left + /// the safe-integer domain. CacheWrite => "cache_write", + /// A tracked write asked for retention above the one-hour physical + /// cap and was clamped: a configuration signal, not a failure. TrackedTtlClamped => "tracked_ttl_clamped", + /// The codec failed or panicked while decoding a stored payload. SerializationLoad => "serialization_load", + /// The codec failed or panicked while encoding a value for the remote layer. SerializationDump => "serialization_dump", + /// zstd compression failed while preparing a remote write. Compression => "compression", + /// An explicit invalidation failed, including one made without a + /// remote adapter. Invalidation => "invalidation", + /// The source failed, panicked or exceeded its deadline; reported + /// with `in_fallback` set. Fallback => "fallback", } } @@ -70,18 +115,35 @@ str_enum! { str_enum! { /// Terminal outcomes of sampled shadow validation. ShadowOutcome { + /// The cached and source values compared equal. Match => "match", + /// They differed, and a confirmation read found the stored payload + /// unchanged. Mismatch => "mismatch", + /// They differed, but the stored payload changed or disappeared + /// before confirmation. Superseded => "superseded", + /// A clean shadow miss was populated. Filled => "filled", + /// A tracked fill was withheld because its stamp did not exceed the + /// watermark observed by the initial read. FillFenced => "fill_fenced", + /// Preparing (encoding, compression) or writing a clean-miss fill failed. FillError => "fill_error", + /// The initial detached remote read failed, including by deadline. RedisError => "redis_error", + /// The source-of-truth read failed. SourceError => "source_error", + /// The retained payload could not be decoded for comparison. DeserializationError => "deserialization_error", + /// The comparator returned an error or panicked. ComparisonError => "comparison_error", + /// The confirmation remote read failed. ConfirmationError => "confirmation_error", + /// The shadow deadline expired before a verdict. Timeout => "timeout", + /// Per-key deduplication or the instance's in-flight cap rejected + /// the job before it started. Dropped => "dropped", } } @@ -89,8 +151,13 @@ str_enum! { str_enum! { /// Terminal outcomes of stale-on-error recovery. RecoveryOutcome { + /// A retained stale frame passed the return-time age check and + /// supplied the result. Served => "served", + /// No retained frame remained eligible: none was kept, or its age + /// reached the ceiling. Miss => "miss", + /// Decoding the retained frame failed. DeserializationError => "deserialization_error", } } @@ -98,38 +165,70 @@ str_enum! { str_enum! { /// Compression outcomes for writes and reads. CompressionOutcome { + /// Write: the payload met the threshold and the marked zstd frame was + /// smaller than the escaped raw form. Compressed => "compressed", + /// Write: the payload was below the threshold and was stored raw. BelowThreshold => "below_threshold", + /// Write: compression ran but did not shrink the payload; stored raw. NotSmaller => "not_smaller", + /// Write: the payload exceeds the decompression cap and was stored raw + /// so a read could still accept it; a capacity signal. WriteOverLimit => "write_over_limit", + /// Read: a marked zstd frame decoded successfully. Decompressed => "decompressed", + /// Read: zstd rejected a marked payload as malformed or truncated; the + /// stored bytes were handed through untouched. FallbackRaw => "fallback_raw", + /// Read: decompressing would exceed the cap; the stored bytes were + /// handed through untouched. An integrity signal. ReadOverLimit => "read_over_limit", } } str_enum! { - SerializationOperation { Dump => "dump", Load => "load" } + /// Which codec direction a serialization timing measures. + SerializationOperation { + /// Encoding a value for the remote layer. + Dump => "dump", + /// Decoding a stored payload. + Load => "load", + } } str_enum! { - CompressionOperation { Compress => "compress", Decompress => "decompress" } + /// Which envelope direction a compression timing measures. + CompressionOperation { + /// zstd compression during a write. + Compress => "compress", + /// zstd decompression during a read. + Decompress => "decompress", + } } /// Labels shared by layer-scoped events. Logical keys are never labels. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Labels { + /// The instance namespace (default `urn`); unrelated to any exporter's + /// metric-name namespace. pub namespace: Arc, + /// The operation name; `watermark` on invalidation error events. pub use_case: Arc, + /// The entity type of the identity. pub key_type: Arc, + /// The layer the event refers to, or [`Layer::Noop`] when none was reached. pub layer: Layer, } /// Labels of outcome-scoped events (shadow validation and recovery). #[derive(Debug, Clone, PartialEq, Eq)] pub struct OutcomeLabels { + /// The instance namespace (default `urn`); unrelated to any exporter's + /// metric-name namespace. pub namespace: Arc, + /// The operation name. pub use_case: Arc, + /// The entity type of the identity. pub key_type: Arc, } @@ -137,87 +236,164 @@ pub struct OutcomeLabels { /// sizes report bytes. #[derive(Debug, Clone, PartialEq)] pub enum Event { + /// One lookup of an enabled cache layer, emitted before its result is known. Request { + /// The layer that was looked up. labels: Labels, }, + /// A layer lookup found no servable value. Miss { + /// The layer that missed. labels: Labels, + /// The bounded cause; request-local and process-local misses are + /// always [`MissReason::ValueAbsent`]. reason: MissReason, }, + /// A layer, or the whole call (`noop`), was skipped. Disabled { + /// The skipped layer, or [`Layer::Noop`] when no layer was reached. labels: Labels, + /// Why it was skipped. reason: DisabledReason, }, + /// A bounded failure site fired. Cache plumbing failures fail open; only + /// source failures reach the caller. Error { + /// The layer whose work failed. labels: Labels, + /// The failure site. error: ErrorKind, + /// `true` when the source (fallback) failed or timed out rather than + /// cache plumbing. in_fallback: bool, }, + /// One explicit invalidation attempt, emitted before the remote call and + /// even when no remote adapter is configured. Invalidation { + /// The namespace whose entity watermark is advanced. namespace: Arc, + /// The entity type being invalidated. key_type: Arc, + /// Always [`Layer::Remote`]: watermarks live only in the remote layer. layer: Layer, }, + /// A caller joined an execution already in flight for the same logical + /// key instead of starting its own. Coalesced { + /// The identity of the joined execution. labels: OutcomeLabels, + /// Whether the flight was registered on the request scope or the instance. scope: CoalescingScope, }, + /// One terminal outcome of an admitted or explicitly dropped shadow job. ShadowValidation { + /// The identity the job validated. labels: OutcomeLabels, + /// The terminal verdict. outcome: ShadowOutcome, }, /// Age in seconds of the validated value at a match or confirmed mismatch, clamped at zero. ShadowValueAge { + /// The identity the job validated. labels: OutcomeLabels, + /// [`ShadowOutcome::Match`] or [`ShadowOutcome::Mismatch`]; no other + /// outcome carries an age. outcome: ShadowOutcome, + /// Observing wall clock minus the frame's `created_at_ms`, never negative. seconds: f64, }, /// Positive offset in seconds of a frame dated after the observing clock. FutureTimestampOffset { + /// The layer whose read decoded the frame. labels: Labels, + /// How far the frame's `created_at_ms` lies ahead of the observing wall clock. seconds: f64, }, + /// One classifier-authorized stale-on-error recovery check after a + /// source failure. StaleRecovery { + /// The identity being recovered. labels: OutcomeLabels, + /// Whether a retained frame was served. outcome: RecoveryOutcome, }, + /// Age of the retained frame at the moment recovery served it; emitted + /// only alongside [`RecoveryOutcome::Served`]. StaleRecoveryValueAge { + /// The identity being recovered. labels: OutcomeLabels, + /// Always [`RecoveryOutcome::Served`]. outcome: RecoveryOutcome, + /// Observing wall clock minus the frame's `created_at_ms`, clamped at zero. seconds: f64, }, + /// One envelope decision: on every write while compression is enabled, + /// and on reads whose payload carried a zstd marker. Compression { + /// The layer performing the write or read. labels: Labels, + /// Write and read outcomes are disjoint sets; see [`CompressionOutcome`]. outcome: CompressionOutcome, }, + /// Lookup latency of one layer, hit or miss; a remote read's value + /// includes any wait up to its deadline. Get { + /// The layer looked up. labels: Labels, + /// Elapsed time of the lookup. seconds: f64, }, + /// Time until the source settled or its deadline elapsed, once per + /// source invocation. Fallback { + /// The deepest layer traversed before the source ran, or + /// [`Layer::Noop`] when caching was bypassed. labels: Labels, + /// Elapsed time from invoking the source until it settled or its + /// deadline fired. seconds: f64, }, + /// Codec latency of one encode or decode, emitted whether or not the + /// codec succeeded. Serialization { + /// The layer the payload was written to or read from. labels: Labels, + /// Encode (`dump`) or decode (`load`). operation: SerializationOperation, + /// Elapsed time inside the codec. seconds: f64, }, + /// Serializer output size of one remote write, before compression and escaping. Size { + /// The layer being written. labels: Labels, + /// Length of the codec output. bytes: u64, }, + /// Prepared payload size of one remote write, after compression and + /// escaping, before dispatch. StoredSize { + /// The layer being written. labels: Labels, + /// Length of the payload handed to the adapter, excluding the + /// ten-byte frame header. bytes: u64, }, + /// Size ratio of one write that chose the compressed form. CompressionRatio { + /// The layer being written. labels: Labels, + /// Marked compressed bytes divided by serializer output bytes. ratio: f64, }, + /// zstd latency: on writes when compression ran (compressed or not + /// smaller), on reads when a marked frame was decoded or rejected. CompressionDuration { + /// The layer performing the write or read. labels: Labels, + /// Compress (write) or decompress (read). operation: CompressionOperation, + /// Elapsed time of the envelope step. seconds: f64, }, } @@ -225,6 +401,8 @@ pub enum Event { /// Receives every public diagnostic. Failures (returned errors or panics) /// never change a cache, source or maintenance result. pub trait Observer: Send + Sync + 'static { + /// Receive one event. Called inline on the cache's own path under panic + /// isolation, so it should return quickly and never block. fn observe(&self, event: &Event); /// Whether this observer consumes shadow validation outcomes. @@ -240,8 +418,11 @@ pub trait Observer: Send + Sync + 'static { /// Bounded details of one confirmed shadow mismatch warning. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ShadowMismatchDetails { + /// The instance namespace of the mismatching identity. pub namespace: Arc, + /// The operation name of the mismatching identity. pub use_case: Arc, + /// The entity type of the mismatching identity. pub key_type: Arc, /// The logical key, bounded to 2 KiB of UTF-8. pub cache_key: String, @@ -254,8 +435,12 @@ pub struct ShadowMismatchDetails { /// Severity of a [`LogEvent`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum LogLevel { + /// Verbose diagnostics; no [`LogEvent`] maps here today. Debug, + /// A fail-open diagnostic: the call still produced a result from another + /// layer or the source. Warn, + /// A whole path is broken: key construction or the process-local store. Error, } @@ -263,20 +448,41 @@ pub enum LogLevel { /// invalidation failure, which is also returned to the caller. #[derive(Debug)] pub enum LogEvent { + /// The key selector failed or panicked, or the identity could not form a + /// key; the source ran uncached. [`LogLevel::Error`]. KeyConstructionFailed(BoxError), + /// The policy provider failed or panicked, or the merged policy was + /// malformed; the source ran uncached. PolicyResolutionFailed(BoxError), + /// The [`LocalStore`](crate::LocalStore) read failed or panicked; the + /// local layer was skipped for the call. [`LogLevel::Error`]. LocalReadFailed(BoxError), + /// The [`LocalStore`](crate::LocalStore) write failed or panicked; the + /// value was still returned. LocalWriteFailed(BoxError), + /// The caller-serving remote read failed or exceeded its read budget. + /// Detached shadow reads do not log. RemoteReadFailed(BoxError), + /// Preparing or dispatching a caller-serving remote write failed: + /// encoding, compression, the write stamp or the adapter. RemoteWriteFailed(BoxError), + /// The stale-recovery classifier panicked; recovery was denied. RecoveryPredicateFailed(BoxError), + /// Decoding the retained stale frame failed during recovery; the source + /// error was returned. RecoveryDecodeFailed(BoxError), + /// A shadow job's clean-miss fill failed while preparing or writing the payload. ShadowFillFailed(BoxError), + /// A confirmed shadow mismatch, logged only when the resolved policy's + /// `log_mismatches` is true. ShadowMismatch(ShadowMismatchDetails), + /// An explicit invalidation failed; the same error is returned to the caller. InvalidationFailed(BoxError), } impl LogEvent { + /// [`LogLevel::Error`] for key construction and process-local read + /// failures, [`LogLevel::Warn`] for everything else. pub fn level(&self) -> LogLevel { match self { LogEvent::KeyConstructionFailed(_) | LogEvent::LocalReadFailed(_) => LogLevel::Error, @@ -311,6 +517,8 @@ impl fmt::Display for LogEvent { /// Receives structured log events. Panics are isolated from cache results. pub trait Logger: Send + Sync + 'static { + /// Receive one event. Called inline under panic isolation; a panic here + /// is swallowed and the cache result is unaffected. fn log(&self, event: &LogEvent); } diff --git a/rust/src/operation.rs b/rust/src/operation.rs index 578bd0c6..20b62929 100644 --- a/rust/src/operation.rs +++ b/rust/src/operation.rs @@ -53,12 +53,20 @@ pub type Preview = Arc Option + Send + Sync>; /// case is declared at the call site instead of being registered. #[derive(Clone)] pub struct Operation { + /// The logical identity; an empty namespace inherits the instance's. pub identity: Identity, + /// Static caching policy, validated before execution. pub policy: Policy, + /// The source deadline. pub budget: SourceBudget, + /// Encodes and decodes values for the remote layer. pub codec: Arc>, + /// Application equality for shadow validation. pub comparator: Comparator, + /// Per-operation stale-recovery classifier. `None` defers to the + /// instance's, which by default admits only the source deadline error. pub should_recover: Option, + /// Renders bounded value previews for mismatch warnings; `None` logs none. pub preview: Option>, } @@ -107,16 +115,20 @@ impl Operation { } } + /// Replace the static policy. pub fn policy(mut self, policy: Policy) -> Self { self.policy = policy; self } + /// Replace the source deadline. pub fn budget(mut self, budget: SourceBudget) -> Self { self.budget = budget; self } + /// Set the stale-recovery classifier for this operation, overriding the + /// instance's. pub fn should_recover( mut self, predicate: impl Fn(&Error) -> bool + Send + Sync + 'static, @@ -125,6 +137,7 @@ impl Operation { self } + /// Set the mismatch-warning preview; `None` from the closure omits the value. pub fn preview( mut self, preview: impl Fn(&T) -> Option + Send + Sync + 'static, diff --git a/rust/src/policy.rs b/rust/src/policy.rs index 11e47f13..44dab1fb 100644 --- a/rust/src/policy.rs +++ b/rust/src/policy.rs @@ -34,6 +34,7 @@ pub struct Policy { pub stale_on_error_max_age_sec: Option, /// Remote read budget in milliseconds, `1..=MAX_DEADLINE_MS`. pub remote_read_timeout_ms: Option, + /// Shadow validation cohort and warning policy. Omitted means no shadow work. pub shadow: Option, } @@ -66,46 +67,58 @@ impl Policy { } } + /// Set whether values are memoized for the outermost enabled scope. pub fn request_local(mut self, enabled: bool) -> Self { self.request_local = Some(enabled); self } + /// Set whether concurrent same-key callers share one execution. pub fn coalesce(mut self, enabled: bool) -> Self { self.coalesce = Some(enabled); self } + /// Set the process-local TTL in whole seconds, enabling the layer with + /// an implied full ramp. pub fn local_ttl_sec(mut self, ttl_sec: u64) -> Self { self.local_ttl_sec = Some(ttl_sec); self } + /// Set the remote TTL in whole seconds, enabling the layer with an + /// implied full ramp. pub fn remote_ttl_sec(mut self, ttl_sec: u64) -> Self { self.remote_ttl_sec = Some(ttl_sec); self } + /// Set the process-local serving cohort percentage, `0.0..=100.0`. pub fn local_ramp(mut self, ramp: f64) -> Self { self.local_ramp = Some(ramp); self } + /// Set the remote serving cohort percentage, `0.0..=100.0`. pub fn remote_ramp(mut self, ramp: f64) -> Self { self.remote_ramp = Some(ramp); self } + /// Set the stale-recovery age ceiling in seconds. Zero disables + /// recovery; a positive value must exceed the remote TTL. pub fn stale_on_error_max_age_sec(mut self, seconds: u64) -> Self { self.stale_on_error_max_age_sec = Some(seconds); self } + /// Set the remote read budget in milliseconds, overriding the instance default. pub fn remote_read_timeout_ms(mut self, ms: u64) -> Self { self.remote_read_timeout_ms = Some(ms); self } + /// Set the shadow validation policy. pub fn shadow(mut self, shadow: ShadowPolicy) -> Self { self.shadow = Some(shadow); self @@ -298,6 +311,8 @@ impl Policy { pub struct RuntimePolicy(pub Value); impl RuntimePolicy { + /// Wrap a JSON overlay as received from a configuration source. Nothing + /// is validated here; resolution judges every leaf per call. pub fn from_json(value: Value) -> Self { RuntimePolicy(value) } @@ -318,6 +333,9 @@ impl From for RuntimePolicy { /// Instance defaults consulted during resolution. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct PolicyDefaults { + /// Instance remote read budget in milliseconds, used when neither the + /// operation nor the overlay sets one. Zero selects the library default + /// of 50 ms. pub remote_read_timeout_ms: u64, } @@ -337,12 +355,15 @@ pub struct PolicyError(pub String); /// One serving layer after resolution. #[derive(Debug, Clone, Copy, PartialEq)] pub struct ResolvedLayer { + /// Whether the layer serves this key: configured and admitted by its ramp. pub enabled: bool, /// Why the layer is disabled, when it is. pub reason: Option, /// A valid TTL and ramp remain available when the ramp excluded this key. pub configured: bool, + /// The effective TTL in milliseconds; zero when not configured. pub ttl_ms: u64, + /// The effective cohort percentage; zero when not configured. pub ramp: f64, } @@ -363,8 +384,13 @@ impl ResolvedLayer { pub struct ResolvedShadow { /// Cohort selection only; admission still needs an eligible path, hook and capacity. pub enabled: bool, + /// The shadow cohort percentage; zero when omitted or invalid. pub ramp: f64, + /// Whether a confirmed mismatch logs a warning; `false` when the flag + /// was omitted or malformed. pub log_mismatches: bool, + /// The shadow ramp leaf was present but invalid: shadow work is off and a + /// `config_resolution` error is emitted on the remote layer. pub config_error: bool, /// Recorded only if a job is admitted. pub logging_config_error: bool, @@ -373,14 +399,24 @@ pub struct ResolvedShadow { /// The captured policy of one enabled invocation. #[derive(Debug, Clone, PartialEq)] pub struct ResolvedPolicy { + /// Whether the call memoizes in the outermost enabled scope. pub request_local: bool, + /// Whether the call joins or leads a single flight for its key. pub coalesce: bool, + /// The process-local layer. pub local: ResolvedLayer, + /// The remote layer. pub remote: ResolvedLayer, + /// The effective remote read budget in milliseconds: overlay, then + /// operation, then instance default. pub remote_read_timeout_ms: u64, /// Zero when recovery is off. pub stale_on_error_max_age_ms: u64, + /// The recovery age leaf was present but invalid (not whole seconds, or + /// not above the remote TTL): recovery is off and a `config_resolution` + /// error is emitted on the remote layer. pub stale_on_error_config_error: bool, + /// The shadow validation policy. pub shadow: ResolvedShadow, } diff --git a/rust/src/preview.rs b/rust/src/preview.rs index ce96b2e5..e7ae5cb9 100644 --- a/rust/src/preview.rs +++ b/rust/src/preview.rs @@ -23,10 +23,12 @@ pub fn clamp_utf8(value: &str, max_bytes: usize) -> String { result } +/// Clamp a logical key to [`SHADOW_LOG_KEY_MAX_BYTES`]. pub fn preview_key(key: &str) -> String { clamp_utf8(key, SHADOW_LOG_KEY_MAX_BYTES) } +/// Clamp a value's JSON preview to [`SHADOW_LOG_VALUE_MAX_BYTES`]. pub fn preview_value(json: &str) -> String { clamp_utf8(json, SHADOW_LOG_VALUE_MAX_BYTES) } diff --git a/rust/src/prometheus.rs b/rust/src/prometheus.rs index 17b4640d..6dda4015 100644 --- a/rust/src/prometheus.rs +++ b/rust/src/prometheus.rs @@ -54,9 +54,11 @@ const FUTURE_TIMESTAMP_OFFSET_BUCKETS: &[f64] = &[ /// The wire schema of one collector: what a scrape exposes. #[derive(Debug, Clone, PartialEq)] pub struct CollectorSchema { + /// The event kind the collector receives. pub kind: MetricKind, /// The fully qualified metric name, prefix included. pub name: String, + /// The Prometheus help text, spelled as in the TypeScript adapter. pub help: &'static str, /// Label names in wire order. pub labels: &'static [&'static str], @@ -65,6 +67,7 @@ pub struct CollectorSchema { } impl CollectorSchema { + /// `true` for the counter vectors; every other collector is a histogram. pub fn is_counter(&self) -> bool { self.kind.is_counter() } @@ -210,13 +213,17 @@ pub enum PrometheusError { /// A collector with this name is already registered by someone else, or /// with another schema. Use a unique prefix or another registry. Conflict { + /// The fully qualified collector name that collided. name: String, + /// The registry's rejection. source: ::prometheus::Error, }, /// The `prometheus` crate rejected a collector definition, which only a /// prefix that is not a valid metric name fragment can cause. InvalidCollector { + /// The fully qualified collector name that was rejected. name: String, + /// The crate's rejection. source: ::prometheus::Error, }, } diff --git a/rust/src/protocol/envelope.rs b/rust/src/protocol/envelope.rs index 014bac0a..cb9ece99 100644 --- a/rust/src/protocol/envelope.rs +++ b/rust/src/protocol/envelope.rs @@ -14,8 +14,12 @@ use crate::observe::CompressionOutcome; use super::frame::ProtocolError; use super::text::replacement_utf8; +/// Envelope byte `0x00`: the rest is raw serializer output whose own first +/// byte would have collided with a marker. pub const MARKER_ESCAPED_RAW: u8 = 0x00; +/// Envelope byte `0x01`: the rest is one zstd frame of UTF-8 text. pub const MARKER_ZSTD_UTF8: u8 = 0x01; +/// Envelope byte `0x02`: the rest is one zstd frame of binary bytes. pub const MARKER_ZSTD_BINARY: u8 = 0x02; const MIN_ZSTD_LEVEL: i32 = 1; @@ -56,16 +60,25 @@ impl CompressionConfig { } } +/// What [`compress_payload`] chose to store, and why. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CompressionWriteResult { + /// The bytes to store: marked zstd output, or the escaped raw form. pub payload: Payload, + /// One of the write outcomes: compressed, below threshold, not smaller + /// or over limit. pub outcome: CompressionOutcome, + /// Length of the serializer output after text replacement, before escaping. pub original_bytes: usize, + /// Length of `payload`. pub stored_bytes: usize, } +/// What [`decompress_payload`] produced from a stored payload. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CompressionReadResult { + /// The decoded payload, or the input unchanged when it passed through or + /// zstd rejected it. pub payload: Payload, /// `None` when the payload passed through untouched. pub outcome: Option, diff --git a/rust/src/protocol/frame.rs b/rust/src/protocol/frame.rs index adff42d8..911ef507 100644 --- a/rust/src/protocol/frame.rs +++ b/rust/src/protocol/frame.rs @@ -14,14 +14,22 @@ use super::text::replacement_utf8; /// Wire-level failures that are errors rather than misses. #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] pub enum ProtocolError { + /// A frame's encoding tag is neither text (`0x00`) nor binary (`0x01`). #[error("Invalid DialCache Redis payload encoding")] PayloadEncoding, + /// The server accepted a command but replied outside the protocol; the + /// text says how, for example a watermark that is not a digit string. #[error("Invalid DialCache Redis reply: {0}")] InvalidReply(String), + /// A stamp is not a nonnegative safe integer, or an invalidation would + /// push its watermark past one. #[error("DialCache timestamp must be a nonnegative safe integer")] InvalidTimestamp, + /// A TTL is not positive, or a TTL or future buffer exceeds 365 days. #[error("DialCache cache TTL must be positive and no greater than 365 days")] InvalidDuration, + /// zstd or its configuration failed on the write side; the text is + /// zstd's error name or the rejected option. #[error("DialCache compression failed: {0}")] Compression(String), } diff --git a/rust/src/redis.rs b/rust/src/redis.rs index ce1470ce..b141497b 100644 --- a/rust/src/redis.rs +++ b/rust/src/redis.rs @@ -105,6 +105,9 @@ pub fn invalidation_script_sha1() -> String { #[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] #[error("Invalid DialCache Redis reply: {message}")] pub struct RedisProtocolError { + /// What the reply violated: an unexpected reply type, a tracked read + /// without exactly two bulk values, a `SET` reply other than `OK`, or an + /// invalidation reply other than the integer `1`. pub message: String, } diff --git a/rust/src/remote.rs b/rust/src/remote.rs index e7e0b0aa..e99027f0 100644 --- a/rust/src/remote.rs +++ b/rust/src/remote.rs @@ -11,6 +11,8 @@ use crate::error::BoxError; pub struct Frame { /// Epoch milliseconds from the writer's clock; a nonnegative safe integer. pub created_at_ms: u64, + /// The stored payload, envelope included; + /// [`decompress_payload`](crate::protocol::decompress_payload) unwraps it. pub payload: Payload, } @@ -30,6 +32,7 @@ pub enum MissReason { } impl MissReason { + /// The `snake_case` label value shared with the TypeScript and Go ports. pub fn as_str(self) -> &'static str { match self { MissReason::ValueAbsent => "value_absent", @@ -39,6 +42,8 @@ impl MissReason { } } + /// The inverse of [`as_str`](Self::as_str). `None` for any other text, + /// which callers treat as [`MissReason::Unclassified`]. pub fn parse(text: &str) -> Option { Some(match text { "value_absent" => MissReason::ValueAbsent, @@ -59,12 +64,16 @@ pub enum ReadResult { /// watermark read atomically with a tracked value, when one existed; a /// refill stamped at or before it is known to remain unreadable. Miss { + /// Why no servable frame was produced. reason: MissReason, + /// The fence described on the variant; always `None` for untracked + /// keys once normalized. observed_watermark_ms: Option, }, } impl ReadResult { + /// A miss that observed no watermark. pub fn miss(reason: MissReason) -> Self { ReadResult::Miss { reason, @@ -72,6 +81,7 @@ impl ReadResult { } } + /// `true` for [`ReadResult::Miss`] of any reason. pub fn is_miss(&self) -> bool { matches!(self, ReadResult::Miss { .. }) } @@ -80,7 +90,10 @@ impl ReadResult { /// Keys of one read: the value key and, for tracked identities, the watermark key. #[derive(Debug, Clone, PartialEq, Eq)] pub struct ReadRequest { + /// The stored value key: the logical key plus the frame suffix. pub value_key: String, + /// The entity watermark key, present only for tracked identities, whose + /// value and watermark must be read atomically. pub watermark_key: Option, } @@ -90,14 +103,20 @@ pub struct ReadRequest { /// to stop waiting, but a dispatched command may still execute. #[derive(Debug, Clone)] pub struct ReadContext { + /// The resolved read budget in milliseconds; the cache stops waiting and + /// cancels the token once it elapses. pub timeout_ms: u64, + /// Cancelled by the cache when the budget elapses; adapters may stop + /// waiting on it. pub cancel: CancelToken, } /// One complete-frame write: a single native `SET value_key frame PX ttl_ms`. #[derive(Debug, Clone, PartialEq, Eq)] pub struct WriteRequest { + /// The stored value key to `SET`. pub value_key: String, + /// The complete frame, stamped by the cache; store it exactly. pub frame: Frame, /// Positive milliseconds no greater than 365 days. pub ttl_ms: u64, @@ -106,6 +125,7 @@ pub struct WriteRequest { /// One invalidation: advance the entity watermark monotonically. #[derive(Debug, Clone, PartialEq, Eq)] pub struct InvalidateRequest { + /// The entity watermark key, `{namespace:keyType:id}#watermark`. pub watermark_key: String, /// The invalidating process's wall clock sample; reused through retries. pub invalidated_at_ms: u64, @@ -123,11 +143,18 @@ pub struct InvalidateRequest { /// [`crate::protocol::decode_frame`] and [`crate::protocol::encode_frame`] /// for the wire format. pub trait Remote: Send + Sync + 'static { + /// Read one value, and for tracked requests its watermark atomically from + /// a primary, then classify the result. An `Err` counts as a `cache_read` + /// error and the call falls through to the source. fn read( &self, request: ReadRequest, context: ReadContext, ) -> BoxFuture<'_, Result>; + /// Store the complete frame under `value_key` with `ttl_ms` retention as + /// one native `SET ... PX`, without touching watermarks. fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>>; + /// Advance the entity watermark to at least + /// `invalidated_at_ms + future_buffer_ms`, only ever widening its retention. fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>>; } diff --git a/rust/src/testing.rs b/rust/src/testing.rs index d6a2697f..f82e3e78 100644 --- a/rust/src/testing.rs +++ b/rust/src/testing.rs @@ -56,6 +56,8 @@ pub struct VirtualClock { } impl VirtualClock { + /// A clock at `wall_ms` epoch milliseconds with zero elapsed and + /// scheduler time. pub fn new(wall_ms: i64) -> Arc { Arc::new(VirtualClock { state: Mutex::new(ClockState { @@ -73,6 +75,7 @@ impl VirtualClock { self.shift_ns(delta_ms as i128 * 1_000_000, wall_only); } + /// [`shift`](Self::shift) in nanoseconds; elapsed time saturates at zero. pub fn shift_ns(&self, delta_ns: i128, wall_only: bool) { let mut state = self.state.lock(); state.wall_ns += delta_ns; @@ -117,6 +120,7 @@ impl VirtualClock { self.state.lock().scheduler_ns } + /// Timers registered and neither fired nor dropped. pub fn pending_timers(&self) -> usize { let state = self.state.lock(); state @@ -198,6 +202,8 @@ pub struct StepRuntime { } impl StepRuntime { + /// A runtime whose timers register on `clock` and whose tasks wait for + /// a [`TestExecutor`] to drain them. pub fn new(clock: Arc) -> Arc { Arc::new(StepRuntime { queues: Mutex::new(Queues::default()), @@ -241,7 +247,9 @@ impl Runtime for StepRuntime { pub struct TestExecutor { pool: LocalPool, local: LocalSpawner, + /// The virtual clock to share with every instance under test. pub clock: Arc, + /// The runtime handle to pass to every instance under test. pub runtime: Arc, } @@ -254,6 +262,8 @@ impl std::fmt::Debug for TestExecutor { } impl TestExecutor { + /// A fresh pool with its clock at `wall_ms`; replayed histories use + /// [`WALL_EPOCH_MS`]. pub fn new(wall_ms: i64) -> Self { let pool = LocalPool::new(); let local = pool.spawner(); @@ -379,6 +389,7 @@ impl std::fmt::Debug for VirtualGridClock { } impl VirtualGridClock { + /// Align to the millisecond grid at `base`'s current elapsed time. pub fn new(base: Arc) -> Self { let now_ns = base.state.lock().elapsed_ns; VirtualGridClock { diff --git a/rust/src/use_case.rs b/rust/src/use_case.rs index 81408549..f4d59624 100644 --- a/rust/src/use_case.rs +++ b/rust/src/use_case.rs @@ -22,11 +22,15 @@ use crate::scope::Scope; /// The key of one call: the entity id and any secondary dimensions. #[derive(Debug, Clone, PartialEq, Default)] pub struct KeySpec { + /// The entity identifier, spelled as text. pub id: String, + /// Secondary dimensions; absent values are dropped and names sorted when + /// the key is built. pub args: Vec<(String, ArgValue)>, } impl KeySpec { + /// A key with `id` and no secondary dimensions. pub fn new(id: impl ToString) -> Self { KeySpec { id: id.to_string(), @@ -181,6 +185,7 @@ where self } + /// Replace the JSON codec; required by [`register_custom`](Self::register_custom). pub fn codec(mut self, codec: Arc>) -> Self { self.codec = Some(codec); self @@ -357,10 +362,12 @@ where Args: Clone + Send + Sync + 'static, T: Send + Sync + 'static, { + /// The operation name given at registration. pub fn use_case(&self) -> &str { &self.inner.use_case } + /// The entity type given at registration. pub fn key_type(&self) -> &str { &self.inner.key_type } From bd95cb95593f8a6b8055d8e25697b99fa56a81fd Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 01:38:23 -0700 Subject: [PATCH 18/31] fix(rust): close scopes on drop, settle dropped flights, share Prometheus collectors by clone, and run the production clock alignment in the local-clock replay Review follow-ups on the Rust port: - DialCache::enable now closes its scope through the RAII guard, so a dropped or unwinding callback future no longer leaves retained Scope clones enabled. - execute hands a scope owned by another instance through to the source unchanged instead of replacing it with Scope::outside(). - Error::is_fallback_timeout also recognises a nested dialcache::Error boxed by a source. - Flight registration and detached work are RAII: a leader task dropped at runtime shutdown unregisters its flight and settles followers with an error, shadow slots free themselves, and start_pending settles its cell from Drop. - LocalStore hands displaced and expired entries back and Owner::close retires its memo outside the lock, so value destructors never run under a cache lock; cancel callbacks are panic-isolated. - TokioRuntime captures a runtime handle at construction; building outside a tokio context is a ConfigError rather than a panic on first use. A tokio integration test covers coalescing, deadlines and the cancellation contracts. - SystemClock::with_sources runs the production grid alignment over caller-supplied sources; the local-clock replay now uses it over the virtual clock instead of a separate test double. - PrometheusObserver drops the address-keyed reuse table: clone one observer per registry, a second registration is a Conflict. - Harness: case timestamps bound the actual run, the settlement control replays each profile once, redis_integration declares its feature. - Validation: cargo runs inside rust/ so rustup honours the toolchain pin; make integration-rust runs the real-server tests and CI calls it; the formal guides describe three replaying ports. --- .github/workflows/ci.yaml | 2 + Makefile | 4 +- formal/PORTING.md | 33 +++--- formal/README.md | 17 ++-- formal/validation.mjs | 20 ++-- go/README.md | 2 +- rust/Cargo.toml | 5 + rust/src/cancel.rs | 7 +- rust/src/clock.rs | 104 ++++++++++++++++--- rust/src/codec.rs | 5 +- rust/src/engine.rs | 91 +++++++++-------- rust/src/error.rs | 10 +- rust/src/execution.rs | 96 +++++++++++++----- rust/src/flight.rs | 36 ++++++- rust/src/lib.rs | 2 +- rust/src/local.rs | 49 ++++++--- rust/src/prometheus.rs | 80 ++++----------- rust/src/runtime.rs | 41 +++++++- rust/src/scope.rs | 15 ++- rust/src/shadow.rs | 45 ++++++--- rust/src/testing.rs | 59 ++++------- rust/tests/conformance.rs | 29 ++++-- rust/tests/formal/driver.rs | 12 +-- rust/tests/formal/local_clock.rs | 9 +- rust/tests/metrics_exporters.rs | 27 ++++- rust/tests/settlement_control.rs | 7 +- rust/tests/tokio_runtime.rs | 168 +++++++++++++++++++++++++++++++ test/formal-rust-replay.test.ts | 29 ++++-- test/formal-validation.test.ts | 12 ++- 29 files changed, 715 insertions(+), 301 deletions(-) create mode 100644 rust/tests/tokio_runtime.rs diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ca33fb73..c1c52ef2 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -77,6 +77,8 @@ jobs: rust: "true" - name: Check Rust formatting, clippy and native tests including the smoke conformance run run: make check-rust + - name: Check real Redis, Valkey and Cluster integrations and invalidation vectors + run: make integration-rust # The existing required `test` status now requires every native language job. test: diff --git a/Makefile b/Makefile index e63e0215..2061e74f 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ NODE ?= node -.PHONY: help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go package-floor ci explore model-check +.PHONY: help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go integration-rust package-floor ci explore model-check -help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go package-floor ci explore model-check: +help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go integration-rust package-floor ci explore model-check: $(NODE) formal/validation.mjs $@ diff --git a/formal/PORTING.md b/formal/PORTING.md index faa2612c..41355cdd 100644 --- a/formal/PORTING.md +++ b/formal/PORTING.md @@ -261,7 +261,7 @@ from the repository root: ```sh make check # Fast native checks and committed smoke; not full acceptance. -make formal # Rust model/corpus checks, then prepared TS and Go replay. +make formal # Quint model/corpus checks, then prepared TS, Go and Rust replay. make model-check # Separate finite symbolic checks; requires Java 21 and tar. make mutations # Measures both fault catalogs over the generated corpus. make integration # Real-server interoperability; requires Docker. @@ -272,8 +272,8 @@ including symbolic checks and the exact Node 22.15.0 package floor. `make formal-check` runs the Quint evidence lane (every scheduled model, its public regressions and the model mutation challenges); the aggregate requires it, but the port lanes do not wait for it. `make formal-generate` produces the -full corpus and the shared witness evidence; `make formal-ts` and -`make formal-go` each prepare and complete one port's run against them. The +full corpus and the shared witness evidence; `make formal-ts`, `make formal-go` +and `make formal-rust` each prepare and complete one port's run against them. The parity and mutation lanes depend only on the generated corpus and shared witness evidence and run in parallel in hosted CI, whose aggregate requires all of them. These are the same entry points used by hosted CI. @@ -286,7 +286,7 @@ complete inventory. A green smoke lane supplies no full-parity claim. Print the current required IDs with `node formal/conformance.mjs inventory`. The commands below describe the lower-level completion API for implementers of another port; the Make -targets already orchestrate it for TS and Go. +targets already orchestrate it for TypeScript, Go and Rust. The shared inventory contains stable language-neutral IDs: @@ -311,35 +311,40 @@ Prepare each port immediately before its native tests. For TS, the low-level command is `node formal/conformance.mjs prepare typescript .formal-traces/ts-context.json`. Run its complete native suite, evaluate the shared witness evidence, and validate its completion before using -`node formal/conformance.mjs prepare go .formal-traces/go-context.json`. -Go preparation binds the witness evidence the shared evaluator just produced. -Do not prepare both contexts consecutively before running either suite. +`node formal/conformance.mjs prepare go .formal-traces/go-context.json` or +`node formal/conformance.mjs prepare rust .formal-traces/rust-context.json`. +Go and Rust preparation bind the witness evidence the shared evaluator just +produced. Do not prepare two contexts consecutively before running either suite. -For another language, supply a JSON array containing every repository-relative +TypeScript, Go and Rust have built-in default source inventories. For another +language, supply a JSON array containing every repository-relative implementation, driver, adapter, dependency-lock and test-configuration file that affects execution: ```sh -node formal/conformance.mjs prepare rust .formal-traces/rust-context.json rust/conformance-sources.json +node formal/conformance.mjs prepare zig .formal-traces/zig-context.json zig/conformance-sources.json ``` These commands do not run the implementation. They record a unique run ID, preparation time, specification/source fingerprints, the exact corpus bytes, and the required case inventory. A port's source manifest is a reviewed input declaration; the checker cannot discover an omitted native dependency itself. -Go's default inputs also include the shared source/fixture definitions and the -evaluated witness JSON under `.formal-traces/go-parity-witnesses/`. Another port -that consumes auxiliary evidence must include those files in its input manifest. +The Go and Rust default inputs also include the shared source/fixture +definitions and the evaluated witness JSON under +`.formal-traces/go-parity-witnesses/`. Another port that consumes auxiliary +evidence must include those files in its input manifest. Run native tests with complete trace directory selectors. Preserve the original assertion report and its timestamps. The supplied report adapters accept -Vitest JSON and `go test -json`, respectively: +Vitest JSON, `go test -json` and the Rust harness's JSON-lines report: ```sh node formal/conformance-adapters.mjs typescript .formal-traces/ts-replay.json .formal-traces/ts-context.json > .formal-traces/ts-completion.json node formal/conformance-adapters.mjs go .formal-traces/go-replay.jsonl .formal-traces/go-context.json > .formal-traces/go-completion.json +node formal/conformance-adapters.mjs rust .formal-traces/rust-replay.jsonl .formal-traces/rust-context.json > .formal-traces/rust-completion.json node formal/conformance.mjs check .formal-traces/ts-completion.json .formal-traces/ts-context.json node formal/conformance.mjs check .formal-traces/go-completion.json .formal-traces/go-context.json +node formal/conformance.mjs check .formal-traces/rust-completion.json .formal-traces/rust-context.json ``` Another language emits the same JSON completion schema: `schemaVersion: 1`, @@ -355,7 +360,7 @@ failed and incomplete results are rejected. So are changed source/corpus bytes and native runs predating preparation. Keep contexts, corpus, original reports and completion JSON together under `.formal-traces/`. Mutation targets validate the relevant full reports against current source and corpus fingerprints before -starting: TS mutations need TS completion, and Go mutations need both ports. +starting: TS mutations need TS completion, and Go mutations need TS and Go completion. Supplied adapters verify actual native assertion records before producing completion results. A completion document is test evidence, not cryptographic attestation that an untrusted driver behaved diff --git a/formal/README.md b/formal/README.md index 86e6a715..e70ada5d 100644 --- a/formal/README.md +++ b/formal/README.md @@ -116,11 +116,11 @@ downloads its pinned solver archive. make help # Targets and prerequisites. make check # Native checks, package, docs and inventories. make smoke # Committed Quint-derived histories in every port. -make formal # Rust model checks, full corpus and both-port completion. +make formal # Quint model checks, full corpus and every port's completion. make differential # Replay composed profiles' reference corpus through the working tree. make model-check # Separate finite symbolic checks; Java 21 and tar required. make mutations # Challenge assertions after full replay has passed. -make integration # Real Redis/Valkey/Cluster and interoperability. +make integration # Real Redis/Valkey/Cluster in every port and interoperability. make explore # Fresh recorded seed in an isolated source snapshot. make ci NODE22_BIN=/absolute/path/to/node22/bin/node ``` @@ -128,12 +128,13 @@ make ci NODE22_BIN=/absolute/path/to/node22/bin/node `make formal-check` is the Quint evidence lane: it typechecks and runs every scheduled model with the Rust evaluator, the public regressions and the model mutation challenges. `make formal-generate` runs generation, fixture -recomputation and the shared witness evaluation; `make formal-ts` and -`make formal-go` then complete each port's replay against that exact corpus. -`make mutations-ts` and `make mutations-go` split the fault campaigns. The -parity and mutation lanes depend only on the generated corpus and shared witness -evidence, so hosted CI runs all four in parallel and none of them waits for the -model check, which runs beside generation; the aggregate requires every lane. +recomputation and the shared witness evaluation; `make formal-ts`, +`make formal-go` and `make formal-rust` then complete each port's replay +against that exact corpus. `make mutations-ts` and `make mutations-go` split +the fault campaigns (the Rust port has no mutation lane yet). The parity and +mutation lanes depend only on the generated corpus and shared witness evidence, +so hosted CI runs all five in parallel and none of them waits for the model +check, which runs beside generation; the aggregate requires every lane. `make fixtures-check` recomputes committed artifacts; after an intentional model edit, update them with `node formal/generate-artifacts.mjs --write` first. `make ci` includes the separate symbolic checks after `make formal`, as well as diff --git a/formal/validation.mjs b/formal/validation.mjs index 1d9e4d8d..f885b8ce 100644 --- a/formal/validation.mjs +++ b/formal/validation.mjs @@ -11,7 +11,7 @@ const aggregateTargets = { check: ['check-ts', 'check-go', 'check-rust', 'docs', 'audit'], formal: ['formal-check', 'formal-generate', 'formal-ts', 'formal-go', 'formal-rust'], mutations: ['mutations-ts', 'mutations-go'], - integration: ['integration-ts', 'integration-go'], + integration: ['integration-ts', 'integration-go', 'integration-rust'], ci: ['check', 'package-floor', 'formal', 'model-check', 'integration', 'mutations'], }; export const targetDescriptions = { @@ -38,9 +38,10 @@ export const targetDescriptions = { 'mutations-go': 'Measure Go semantic mutations over the generated corpus and shared witness evidence (MUTATION_SHARD=/ measures one shard)', 'mutations-merge-ts': 'Merge TypeScript mutation shards into the complete report; refuses inconsistent or missing shards', 'mutations-merge-go': 'Merge Go mutation shards into the complete report; refuses inconsistent or missing shards', - integration: 'Run real TypeScript and Go Redis/Valkey/Cluster integration checks', + integration: 'Run real TypeScript, Go and Rust Redis/Valkey/Cluster integration checks', 'integration-ts': 'Run TypeScript real integration checks', 'integration-go': 'Run Go real integration and interoperability checks with race detection', + 'integration-rust': 'Run Rust real Redis/Valkey/Cluster integration checks and invalidation vector replay', 'package-floor': 'Check zstd and the packed package on exact Node 22.15.0 (NODE22_BIN)', ci: 'Run check, package-floor, formal, model-check, integration and mutations in dependency order', }; @@ -96,7 +97,9 @@ export function validationPlan(target, { directory = root, environment = process const pnpm = (label, ...args) => ({ label, command: 'corepack', args: ['pnpm', ...args] }); const go = (label, ...args) => ({ label, command: 'go', args: ['-C', 'go', ...args] }); // Every step spawns from the repository root; the manifest path selects the crate. - const cargo = (label, subcommand, ...args) => ({ label, command: 'cargo', args: [subcommand, '--manifest-path', 'rust/Cargo.toml', ...args] }); + // Cargo runs inside rust/ so rustup resolves rust/rust-toolchain.toml; the + // crate's tests locate the repository through CARGO_MANIFEST_DIR, not cwd. + const cargo = (label, subcommand, ...args) => ({ label, command: 'cargo', args: [subcommand, ...args], cwd: 'rust' }); const reportPath = (language, suffix) => `.formal-traces/${language}-${suffix}.json`; const completion = language => ({ ...node(`Validate current ${language} completion`, 'formal/conformance.mjs', 'check', reportPath(language, 'completion'), reportPath(language, 'context')), failureHint: 'A current complete replay is required. Run make formal first; missing or stale reports cannot be reused.' }); @@ -184,6 +187,9 @@ export function validationPlan(target, { directory = root, environment = process 'mutations-merge-go': [node('Merge Go mutation shards', 'formal/merge-mutation-reports.mjs', 'go')], 'integration-ts': [pnpm('Run TypeScript Redis/Valkey/Cluster integrations', 'test:integration')], 'integration-go': [{ ...go('Run Go Redis/Valkey/Cluster and TypeScript interoperability', 'test', '-race', '-tags', 'integration', '-count=1', '-run', '^TestRedisIntegration$', '-json', './...'), stdoutFile: '.formal-traces/go-integration.jsonl' }], + // The Rust integration binary skips itself unless this variable is set, so + // a plain cargo test never needs Docker. + 'integration-rust': [{ ...cargo('Run Rust Redis/Valkey/Cluster integrations', 'test', '--all-features', '--test', 'redis_integration'), env: { DIALCACHE_RUST_INTEGRATION: '1' } }], 'package-floor': [{ label: 'Require a built package for floor checks', requireFile: 'dist/index.js', failureHint: 'Build first with make check-ts, or run make ci with NODE22_BIN set.' }, { label: 'Check Node 22.15 zstd round trip and output ceiling', command: node22, args: ['--eval', floorSmoke], env: { PATH: floorEnvironment(environment, node22).PATH } }, { label: 'Check packed package on Node 22.15', command: node22, args: ['scripts/test-package.mjs'], env: { PATH: floorEnvironment(environment, node22).PATH } }], @@ -208,9 +214,9 @@ export function checkPrerequisites(target, { directory = root, environment = pro const version = probe('go', ['version'], { directory, environment }); if (!/^go version go1\.27\.1\s/.test(version)) throw new Error(`Validation requires Go 1.27.1; found ${version}. Put the pinned Go toolchain on PATH.`); } - if (targets.some(name => ['check-rust', 'smoke', 'formal-rust'].includes(name))) { - const version = probe('cargo', ['--version'], { directory, environment }); - if (!/^cargo 1\.98\.1(?:\s|$)/.test(version)) throw new Error(`Validation requires cargo 1.98.1; found ${version}. Put the pinned Rust toolchain (rust/rust-toolchain.toml) on PATH.`); + if (targets.some(name => ['check-rust', 'smoke', 'formal-rust', 'integration-rust'].includes(name))) { + const version = probe('cargo', ['--version'], { directory: resolve(directory, 'rust'), environment }); + if (!/^cargo 1\.98\.1(?:\s|$)/.test(version)) throw new Error(`Validation requires cargo 1.98.1; found ${version}. Install Rust 1.98.1 (rustup reads rust/rust-toolchain.toml) and put it on PATH.`); } if (targets.some(name => ['formal-check', 'formal-generate', 'fixtures-check', 'explore', 'model-check', 'differential'].includes(name))) { const requiredQuint = JSON.parse(readFileSync(resolve(directory, 'formal/generated-fixtures.lock.json'), 'utf8')).quintVersion; @@ -331,7 +337,7 @@ export async function executeSteps(steps, { directory = root, environment = proc } try { await new Promise((resolveRun, reject) => { - const child = spawn(step.command, step.args, { cwd: directory, env: cleanEnvironment(baseEnvironment, step.env), + const child = spawn(step.command, step.args, { cwd: step.cwd ? resolve(directory, step.cwd) : directory, env: cleanEnvironment(baseEnvironment, step.env), stdio: ['inherit', output ?? (step.requireEmptyStdout ? 'pipe' : 'inherit'), 'inherit'] }); let unexpectedOutput = ''; if (step.requireEmptyStdout) child.stdout.on('data', data => { unexpectedOutput += data; }); diff --git a/go/README.md b/go/README.md index 4c9c3cb5..33bd7875 100644 --- a/go/README.md +++ b/go/README.md @@ -178,7 +178,7 @@ once, then: ```sh make check-go # Native checks, committed cases/smoke and race detection. -make formal # Rust model/corpus checks, then prepared TS and Go replay. +make formal # Quint model/corpus checks, then prepared TS, Go and Rust replay. make model-check # Separate finite symbolic checks. make mutations-go # Go fault catalog over the generated corpus and witness evidence. make integration-go diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 91a5e42a..c83b3b90 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -44,3 +44,8 @@ serde_json = "1" name = "conformance" path = "tests/conformance.rs" harness = false + +[[test]] +name = "redis_integration" +path = "tests/redis_integration.rs" +required-features = ["redis"] diff --git a/rust/src/cancel.rs b/rust/src/cancel.rs index f53f7795..8126183b 100644 --- a/rust/src/cancel.rs +++ b/rust/src/cancel.rs @@ -1,6 +1,7 @@ //! Cooperative cancellation for remote reads. use std::future::Future; +use std::panic::{catch_unwind, AssertUnwindSafe}; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll, Waker}; @@ -57,8 +58,10 @@ impl CancelToken { for waker in wakers { waker.wake(); } + // A callback belongs to an adapter; its panic must not become the + // outcome of the read that is being abandoned. for callback in callbacks { - callback(); + let _ = catch_unwind(AssertUnwindSafe(callback)); } } @@ -74,7 +77,7 @@ impl CancelToken { } } if let Some(callback) = callback { - callback(); + let _ = catch_unwind(AssertUnwindSafe(callback)); } } diff --git a/rust/src/clock.rs b/rust/src/clock.rs index b99bf871..29d31d90 100644 --- a/rust/src/clock.rs +++ b/rust/src/clock.rs @@ -1,6 +1,7 @@ //! Wall and elapsed clocks. -use std::sync::OnceLock; +use std::fmt; +use std::sync::{Arc, OnceLock}; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; /// Separates wall timestamps from monotonic elapsed time. @@ -30,41 +31,114 @@ pub fn grid_origin_ns(now_ns: u128) -> u128 { now_ns - now_ns % 1_000_000 } +type MonotonicSource = Arc Duration + Send + Sync>; +type WallSource = Arc i64 + Send + Sync>; + /// The default clock: system wall time and a monotonic origin aligned to the /// process-wide millisecond grid, so default instances share one local expiry grid. -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct SystemClock { - origin: Instant, + monotonic: MonotonicSource, + wall: WallSource, + /// Nanoseconds from the monotonic source's zero to this clock's origin. + origin_ns: u128, } impl SystemClock { + /// System sources: `SystemTime` for wall time and the `Instant` elapsed + /// since the first default clock of this process for monotonic time. pub fn new() -> Self { let process = *PROCESS_ORIGIN.get_or_init(Instant::now); - let now = Instant::now(); - let since_process = now.saturating_duration_since(process).as_nanos(); - let phase = since_process - grid_origin_ns(since_process); - let origin = now - .checked_sub(Duration::from_nanos(phase as u64)) - .unwrap_or(now); - SystemClock { origin } + Self::with_sources(move || process.elapsed(), system_wall_ms) + } + + /// The production alignment over caller-supplied sources. + /// + /// `monotonic` reads elapsed time since one fixed instant shared by every + /// clock built over it; this clock's origin is the nearest whole + /// millisecond of that reading at or before construction, so clocks built + /// at different fractional times share one expiry grid. `wall` supplies + /// epoch milliseconds. Controlled tests use this to run the default + /// alignment over a virtual clock. + pub fn with_sources( + monotonic: impl Fn() -> Duration + Send + Sync + 'static, + wall: impl Fn() -> i64 + Send + Sync + 'static, + ) -> Self { + let since_process = monotonic().as_nanos(); + SystemClock { + monotonic: Arc::new(monotonic), + wall: Arc::new(wall), + origin_ns: grid_origin_ns(since_process), + } } } +fn system_wall_ms() -> i64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_millis().min(i64::MAX as u128) as i64) + .unwrap_or(0) +} + impl Default for SystemClock { fn default() -> Self { Self::new() } } +impl fmt::Debug for SystemClock { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("SystemClock") + .field("origin_ns", &self.origin_ns) + .finish_non_exhaustive() + } +} + impl Clock for SystemClock { fn wall_ms(&self) -> i64 { - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_millis().min(i64::MAX as u128) as i64) - .unwrap_or(0) + (self.wall)() } fn elapsed(&self) -> Duration { - self.origin.elapsed() + let now_ns = (self.monotonic)().as_nanos(); + Duration::from_nanos(now_ns.saturating_sub(self.origin_ns).min(u64::MAX as u128) as u64) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use parking_lot::Mutex; + + #[test] + fn clocks_built_at_different_fractions_share_one_grid() { + let now = Arc::new(Mutex::new(Duration::from_micros(700))); + let source = { + let now = now.clone(); + move || *now.lock() + }; + let first = SystemClock::with_sources(source.clone(), || 0); + *now.lock() = Duration::from_micros(1_300); + let second = SystemClock::with_sources(source, || 0); + *now.lock() = Duration::from_micros(1_000_900); + // Origins are 0 ms and 1 ms: the same grid, one millisecond apart. + assert_eq!(first.elapsed_ms(), 1_000); + assert_eq!(second.elapsed_ms(), 999); + *now.lock() = Duration::from_micros(1_001_000); + assert_eq!(second.elapsed_ms(), 1_000); + } + + #[test] + fn default_clock_is_monotonic_and_shares_the_process_origin() { + let first = SystemClock::new(); + let second = SystemClock::new(); + let a = first.elapsed(); + assert!(first.elapsed() >= a); + // Both origins lie on the process grid: whole milliseconds after the + // process origin, so the two instances share one expiry grid. + assert_eq!(first.origin_ns % 1_000_000, 0); + assert_eq!(second.origin_ns % 1_000_000, 0); + assert!(second.origin_ns >= first.origin_ns); + assert!(first.wall_ms() > 1_700_000_000_000); } } diff --git a/rust/src/codec.rs b/rust/src/codec.rs index 95875174..3fe78288 100644 --- a/rust/src/codec.rs +++ b/rust/src/codec.rs @@ -88,8 +88,9 @@ pub const JSON_UNDEFINED_SENTINEL: &str = "__dialcache_json_undefined_v1__"; /// /// Values written by TypeScript's default serializer decode as long as they /// are valid JSON for the destination type. The TypeScript `undefined` -/// sentinel decodes as JSON `null`, so an `Option` destination reads it as -/// `None` and any other destination fails open to the source. +/// sentinel decodes as JSON `null`: an `Option` destination reads it as +/// `None`, a destination that accepts `null` (such as `serde_json::Value`) +/// decodes it, and any other destination fails open to the source. #[derive(Debug, Clone, Copy, Default)] pub struct JsonCodec; diff --git a/rust/src/engine.rs b/rust/src/engine.rs index 6a7b8069..a381e271 100644 --- a/rust/src/engine.rs +++ b/rust/src/engine.rs @@ -4,22 +4,21 @@ use std::collections::{HashMap, HashSet}; use std::future::Future; use std::num::NonZeroUsize; use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use std::time::Duration; use futures::future::BoxFuture; use parking_lot::Mutex; use crate::clock::Clock; -use crate::error::{BoxError, ConfigError, Error, SharedError}; +use crate::error::{BoxError, ConfigError, Error}; use crate::flight::{start_pending, Flight, Settled, ValueResult}; use crate::identity::Identity; use crate::limits::{ DEFAULT_LOCAL_CAPACITY, DEFAULT_REMOTE_READ_TIMEOUT_MS, DEFAULT_SHADOW_MAX_IN_FLIGHT, MAX_DEADLINE_MS, MAX_SAFE_INTEGER, MAX_SUPPORTED_DURATION_MS, WATERMARK_USE_CASE, }; -use crate::local::{LocalEntry, LocalStore, LruLocalStore, StoredValue}; +use crate::local::{LocalEntry, LocalRead, LocalStore, LruLocalStore, StoredValue}; use crate::observe::{Event, Labels, Layer, LogEvent, Logger, Observer, OutcomeLabels}; use crate::operation::{downcast_value, erase_load, ErasedOperation, Operation, RecoveryPredicate}; use crate::policy::RuntimePolicy; @@ -87,13 +86,21 @@ impl Core { // Read the clock before taking the lock: it is the only external code // on this path, and a failure must not poison shared state. let now_ms = self.clock.elapsed_ms(); - let mut state = self.state.lock(); - match state.local.as_mut() { - None => Ok(None), - Some(local) => catch_unwind(AssertUnwindSafe(|| local.get(key, now_ms))) - .unwrap_or_else(|payload| { - Err(crate::flight::panic_message(payload).to_string().into()) - }), + let read = { + let mut state = self.state.lock(); + match state.local.as_mut() { + None => return Ok(None), + Some(local) => catch_unwind(AssertUnwindSafe(|| local.get(key, now_ms))) + .unwrap_or_else(|payload| { + Err(crate::flight::panic_message(payload).to_string().into()) + }), + } + }; + // An expired entry drops here, outside the lock: a value's destructor + // may call back into the cache. + match read? { + LocalRead::Live(value) => Ok(Some(value)), + LocalRead::Absent | LocalRead::Expired(_) => Ok(None), } } @@ -104,20 +111,25 @@ impl Core { ttl_ms: u64, ) -> Result<(), BoxError> { let now_ms = self.clock.elapsed_ms(); - let mut state = self.state.lock(); - match state.local.as_mut() { - None => Ok(()), - Some(local) => { - let entry = LocalEntry { - value, - inserted_ms: now_ms, - ttl_ms: ttl_ms.min(i64::MAX as u64) as i64, - }; - catch_unwind(AssertUnwindSafe(|| local.put(key.to_string(), entry))).unwrap_or_else( - |payload| Err(crate::flight::panic_message(payload).to_string().into()), - ) + let displaced = { + let mut state = self.state.lock(); + match state.local.as_mut() { + None => return Ok(()), + Some(local) => { + let entry = LocalEntry { + value, + inserted_ms: now_ms, + ttl_ms: ttl_ms.min(i64::MAX as u64) as i64, + }; + catch_unwind(AssertUnwindSafe(|| local.put(key.to_string(), entry))) + .unwrap_or_else(|payload| { + Err(crate::flight::panic_message(payload).to_string().into()) + }) + } } - } + }; + // The displaced entry drops here, outside the lock. + displaced.map(|_| ()) } } @@ -389,7 +401,7 @@ fn default_clock() -> Result, ConfigError> { #[cfg(feature = "tokio")] fn default_runtime() -> Result, ConfigError> { - Ok(Arc::new(crate::runtime::TokioRuntime)) + Ok(Arc::new(crate::runtime::TokioRuntime::current()?)) } #[cfg(not(feature = "tokio"))] @@ -461,20 +473,17 @@ impl DialCache { } /// Open the outermost enabled scope for `f`. The scope closes when the - /// returned future completes; retained clones then pass through. + /// returned future completes, is dropped before completion, or unwinds; + /// retained clones then pass through. pub async fn enable(&self, f: F) -> R where F: FnOnce(Scope) -> Fut, Fut: Future, { - let owner = Owner::new(); - let scope = Scope { - cache_id: self.core.id, - owner: Some(owner.clone()), - enabled: true, - }; + let guard = self.enable_guard(); + let scope = guard.scope().clone(); let result = f(scope).await; - owner.close(); + drop(guard); result } @@ -686,12 +695,10 @@ impl DialCache { if operation.identity.namespace.is_empty() { operation.identity.namespace = self.core.namespace.to_string(); } + // A scope owned by another instance disables this call but still + // reaches the source, so nested calls on its own instance keep caching. let core = self.core.clone(); - let scope = if scope.cache_id == core.id { - scope.clone() - } else { - Scope::outside() - }; + let scope = scope.clone(); let pending: Settled = start_pending( core.runtime.clone().as_ref(), crate::execution::run(core, scope, Arc::new(operation)), @@ -738,11 +745,3 @@ pub(crate) fn outcome_labels(identity: &Identity) -> OutcomeLabels { key_type: Arc::from(identity.key_type.as_str()), } } - -#[allow(dead_code)] -pub(crate) fn shared_error(error: BoxError) -> SharedError { - Arc::from(error) -} - -#[allow(dead_code)] -pub(crate) struct Unused(AtomicBool, Duration); diff --git a/rust/src/error.rs b/rust/src/error.rs index ed5cc60f..650880b9 100644 --- a/rust/src/error.rs +++ b/rust/src/error.rs @@ -42,11 +42,17 @@ impl Error { } /// Whether this is a source deadline error: the library's own, or one a - /// source propagated from a nested cached call. + /// source propagated from a nested cached call, either as the bare + /// [`FallbackTimeout`] or boxed inside a nested [`Error`]. pub fn is_fallback_timeout(&self) -> bool { match self { Error::FallbackTimeout(_) => true, - Error::Source(error) => error.downcast_ref::().is_some(), + Error::Source(error) => { + error.downcast_ref::().is_some() + || error + .downcast_ref::() + .is_some_and(Error::is_fallback_timeout) + } _ => false, } } diff --git a/rust/src/execution.rs b/rust/src/execution.rs index 660f67fd..58cfbcbf 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -14,7 +14,7 @@ use crate::cancel::CancelToken; use crate::deadline::{await_deadline, seconds_since}; use crate::engine::Core; use crate::error::{BoxError, Error, FallbackTimeout, RemoteReadTimeout, SharedError}; -use crate::flight::{panic_message, start_pending, Flight, Settled, ValueResult}; +use crate::flight::{panic_message, start_pending, Flight, Settled, ValueResult, DROPPED_MESSAGE}; use crate::identity::{Identity, Keys}; use crate::limits::{MAX_DECOMPRESSED_BYTES, MAX_SAFE_INTEGER, MAX_TRACKED_VALUE_TTL_MS}; use crate::local::StoredValue; @@ -37,6 +37,67 @@ pub(crate) type RawRead = Result; #[error("DialCache callback panicked: {0}")] pub(crate) struct PanicError(pub Arc); +/// The registered leader of one key in the request or process flight table. +/// +/// Dropping it before [`Leader::finish`] means the leader's detached task was +/// dropped unpolled (the runtime shut down while the cache outlived it); the +/// flight is then unregistered and settled with an error so later callers, +/// possibly on another runtime, start a fresh execution instead of joining a +/// flight that can never complete. +struct Leader { + core: Arc, + owner: Option>, + key: String, + flight: Arc, + settled: bool, +} + +impl Leader { + fn unregister(&self) { + match &self.owner { + Some(owner) => { + let mut state = owner.state.lock(); + if state + .flights + .get(&self.key) + .is_some_and(|f| Arc::ptr_eq(f, &self.flight)) + { + state.flights.remove(&self.key); + } + } + None => { + let mut state = self.core.state.lock(); + if state + .flights + .get(&self.key) + .is_some_and(|f| Arc::ptr_eq(f, &self.flight)) + { + state.flights.remove(&self.key); + } + } + } + } + + /// Unregister first, then settle: a caller arriving between the two + /// starts a new flight rather than joining a settled one. + fn finish(&mut self, result: ValueResult) { + self.settled = true; + self.unregister(); + self.flight.result.settle(result); + } +} + +impl Drop for Leader { + fn drop(&mut self) { + if !self.settled { + self.unregister(); + self.flight + .result + .settle(Err(Error::Panic(Arc::from(DROPPED_MESSAGE)))); + } + } +} + /// One admitted enabled call. pub(crate) struct Execution { pub(crate) core: Arc, @@ -428,35 +489,18 @@ impl Execution { Some(Err(flight)) => flight, } }; + let mut leader = Leader { + core: self.core.clone(), + owner, + key, + flight, + settled: false, + }; let result = match AssertUnwindSafe(run()).catch_unwind().await { Ok(result) => result, Err(payload) => Err(Error::Panic(panic_message(payload))), }; - { - match &owner { - Some(owner) => { - let mut state = owner.state.lock(); - if state - .flights - .get(&key) - .is_some_and(|f| Arc::ptr_eq(f, &flight)) - { - state.flights.remove(&key); - } - } - None => { - let mut state = self.core.state.lock(); - if state - .flights - .get(&key) - .is_some_and(|f| Arc::ptr_eq(f, &flight)) - { - state.flights.remove(&key); - } - } - } - flight.result.settle(result.clone()); - } + leader.finish(result.clone()); result } diff --git a/rust/src/flight.rs b/rust/src/flight.rs index 13730b91..93db895d 100644 --- a/rust/src/flight.rs +++ b/rust/src/flight.rs @@ -103,9 +103,30 @@ pub(crate) fn panic_message(payload: Box) -> Arc } } +/// The message settled into a cell whose detached task was dropped before it +/// completed: the runtime shut down while the cache outlived it. +pub(crate) const DROPPED_MESSAGE: &str = + "DialCache detached work was dropped before it settled (runtime shut down)"; + +/// Settles the cell from `Drop` when the spawned task is dropped unpolled or +/// mid-await, so waiters on another runtime never hang on a dead task. +struct SettleOnDrop) -> T> { + cell: Settled, + on_panic: Option

, +} + +impl) -> T> Drop for SettleOnDrop { + fn drop(&mut self) { + if let Some(on_panic) = self.on_panic.take() { + self.cell.settle(on_panic(Arc::from(DROPPED_MESSAGE))); + } + } +} + /// Start `work` as detached raw work. The returned cell settles with its -/// result, or with `on_panic` when it panics. Callers that stop waiting keep -/// no ownership of the work. +/// result, with `on_panic` when it panics, and with `on_panic` and +/// [`DROPPED_MESSAGE`] when the runtime drops the task before it completes. +/// Callers that stop waiting keep no ownership of the work. pub(crate) fn start_pending( runtime: &dyn Runtime, work: F, @@ -116,13 +137,18 @@ where F: Future + Send + 'static, { let settled = Settled::new(); - let cell = settled.clone(); + let mut guard = SettleOnDrop { + cell: settled.clone(), + on_panic: Some(on_panic), + }; runtime.spawn(Box::pin(async move { let outcome = match AssertUnwindSafe(work).catch_unwind().await { Ok(value) => value, - Err(payload) => on_panic(panic_message(payload)), + Err(payload) => (guard.on_panic.take().expect("armed guard"))(panic_message(payload)), }; - cell.settle(outcome); + // Disarm before settling so the cell settles exactly once. + guard.on_panic = None; + guard.cell.settle(outcome); })); settled } diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 61b0c30f..18bf62a3 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -53,7 +53,7 @@ pub use engine::{ }; pub use error::{BoxError, ConfigError, Error, FallbackTimeout, RemoteReadTimeout, SharedError}; pub use identity::{normalize_args, ArgValue, Identity, IdentityError, Keys}; -pub use local::{LocalEntry, LocalStore, LruLocalStore, StoredValue}; +pub use local::{LocalEntry, LocalRead, LocalStore, LruLocalStore, StoredValue}; pub use metrics::MetricKind; pub use observe::{Event, Labels, LogEvent, LogLevel, Logger, Observer, ShadowMismatchDetails}; pub use operation::{Comparator, Operation, Preview, RecoveryPredicate, SourceBudget}; diff --git a/rust/src/local.rs b/rust/src/local.rs index 9b5ef017..c7fa3284 100644 --- a/rust/src/local.rs +++ b/rust/src/local.rs @@ -25,11 +25,29 @@ pub struct LocalEntry { /// against whole elapsed milliseconds, promotes on read and write, and never /// renews a TTL on read. Implementations must preserve those rules. pub trait LocalStore: Send + 'static { - /// Return a live entry's value, promoting it. Expired entries are removed - /// and reported as absent without promotion. - fn get(&mut self, key: &str, now_ms: i64) -> Result, BoxError>; + /// Read one key. A live entry is promoted and returned; an expired entry + /// is removed without promotion and handed back so the cache can drop it + /// outside its lock. + fn get(&mut self, key: &str, now_ms: i64) -> Result; /// Insert or replace an entry, promoting it and evicting at capacity. - fn put(&mut self, key: String, entry: LocalEntry) -> Result<(), BoxError>; + /// Returns the displaced entry (the replaced value or the evicted tail), + /// which the cache drops outside its lock. + fn put(&mut self, key: String, entry: LocalEntry) -> Result, BoxError>; +} + +/// The outcome of one [`LocalStore::get`]. +/// +/// Removed entries travel back to the cache instead of dropping inside the +/// store, because a value's destructor may call back into the cache and the +/// store runs under the cache's lock. +#[derive(Debug)] +pub enum LocalRead { + /// No entry under the key. + Absent, + /// The entry had expired and was removed. + Expired(LocalEntry), + /// A live entry, promoted. + Live(StoredValue), } /// The default LRU store. @@ -55,23 +73,30 @@ impl LruLocalStore { } impl LocalStore for LruLocalStore { - fn get(&mut self, key: &str, now_ms: i64) -> Result, BoxError> { + fn get(&mut self, key: &str, now_ms: i64) -> Result { // Peek, check freshness, then promote: an expired entry leaves the LRU // order untouched apart from its own removal. let expired = match self.entries.peek(key) { - None => return Ok(None), + None => return Ok(LocalRead::Absent), Some(entry) => now_ms.saturating_sub(entry.inserted_ms) >= entry.ttl_ms, }; if expired { - self.entries.pop(key); - return Ok(None); + return Ok(match self.entries.pop(key) { + Some(entry) => LocalRead::Expired(entry), + None => LocalRead::Absent, + }); } - Ok(self.entries.get(key).map(|entry| entry.value.clone())) + Ok(match self.entries.get(key) { + Some(entry) => LocalRead::Live(entry.value.clone()), + None => LocalRead::Absent, + }) } - fn put(&mut self, key: String, entry: LocalEntry) -> Result<(), BoxError> { - self.entries.put(key, entry); - Ok(()) + fn put(&mut self, key: String, entry: LocalEntry) -> Result, BoxError> { + Ok(self + .entries + .push(key, entry) + .map(|(_, displaced)| displaced)) } } diff --git a/rust/src/prometheus.rs b/rust/src/prometheus.rs index 17b4640d..9d3ac8a0 100644 --- a/rust/src/prometheus.rs +++ b/rust/src/prometheus.rs @@ -4,28 +4,21 @@ //! [`Registry`] under the names, help text, label sets and histogram buckets //! of the TypeScript adapter, and feeds every [`Event`] to them. //! -//! # Reuse +//! # Sharing one registry //! -//! Collectors are process-wide state: constructing a second observer for the -//! same registry and prefix reuses the collectors the first one created, so -//! observations from both accumulate in one series set. The `prometheus` -//! crate reports a duplicate registration without a handle to the existing -//! collector, so reuse is keyed by the registry's address and the prefix in a -//! process-wide table. Pass the same [`Registry`] value each time, not a -//! clone: a clone shares the registered collectors but has another address, -//! and its construction fails as an externally owned duplicate. -//! -//! An externally registered collector with one of the DialCache names is -//! reported as [`PrometheusError::Conflict`]; the constructor then leaves the -//! registry as it found it. +//! One observer owns one set of collectors. Instances that export to the same +//! registry share the observer: it is `Clone`, and every clone feeds the same +//! series. The `prometheus` crate reports a duplicate registration without a +//! handle to the existing collector, so a second `new` for the same registry +//! and prefix is a [`PrometheusError::Conflict`], as is an externally +//! registered collector under one of the DialCache names; the constructor +//! then leaves the registry as it found it. -use std::collections::HashMap; use std::fmt; -use std::sync::{Arc, LazyLock}; +use std::sync::Arc; use ::prometheus::core::Collector; use ::prometheus::{HistogramOpts, HistogramVec, IntCounterVec, Opts, Registry}; -use parking_lot::Mutex; use crate::metrics::MetricKind; use crate::observe::{Event, Observer}; @@ -207,8 +200,9 @@ pub fn schemas(prefix: &str) -> Vec { /// Construction failures of a [`PrometheusObserver`]. #[derive(Debug)] pub enum PrometheusError { - /// A collector with this name is already registered by someone else, or - /// with another schema. Use a unique prefix or another registry. + /// A collector with this name is already registered: by another observer + /// (clone that observer instead of constructing a second one), by someone + /// else, or with another schema. Use a unique prefix or another registry. Conflict { name: String, source: ::prometheus::Error, @@ -311,34 +305,13 @@ impl Group { } Ok(()) } - - /// Make sure every collector is registered on `registry`, accepting the - /// ones already there. - fn ensure_registered(&self, registry: &Registry, prefix: &str) -> Result<(), PrometheusError> { - for schema in schemas(prefix) { - match registry.register(self.vectors[schema.kind.index()].boxed()) { - Ok(()) | Err(::prometheus::Error::AlreadyReg) => {} - Err(source) => { - return Err(PrometheusError::Conflict { - name: schema.name, - source, - }) - } - } - } - Ok(()) - } } -type GroupKey = (usize, String); - -/// Collector groups by registry address and prefix. Groups are retained for -/// the process lifetime so a later observer for the same registry finds the -/// collectors even after every earlier observer was dropped. -static GROUPS: LazyLock>>> = - LazyLock::new(|| Mutex::new(HashMap::new())); - /// Publishes DialCache diagnostics to Prometheus collectors. +/// +/// Clones share the collectors; hand one observer to every instance that +/// exports to the same registry. +#[derive(Clone)] pub struct PrometheusObserver { group: Arc, prefix: String, @@ -353,23 +326,12 @@ impl fmt::Debug for PrometheusObserver { } impl PrometheusObserver { - /// Create or reuse the DialCache collectors named `dialcache_*` - /// on `registry`. See the module documentation for the reuse rule. + /// Register the DialCache collectors named `dialcache_*` on + /// `registry`. Registering the same names twice on one registry is a + /// [`PrometheusError::Conflict`]: clone the observer instead. pub fn new(registry: &Registry, prefix: &str) -> Result { - let key: GroupKey = (registry as *const Registry as usize, prefix.to_string()); - let mut groups = GROUPS.lock(); - let group = match groups.get(&key) { - Some(group) => { - group.ensure_registered(registry, prefix)?; - group.clone() - } - None => { - let group = Arc::new(Group::build(prefix)?); - group.register(registry, prefix)?; - groups.insert(key, group.clone()); - group - } - }; + let group = Arc::new(Group::build(prefix)?); + group.register(registry, prefix)?; Ok(PrometheusObserver { group, prefix: prefix.to_string(), diff --git a/rust/src/runtime.rs b/rust/src/runtime.rs index 9514c7a4..5a6bde2f 100644 --- a/rust/src/runtime.rs +++ b/rust/src/runtime.rs @@ -23,18 +23,51 @@ pub trait Runtime: Send + Sync + 'static { fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()>; } -/// Spawns onto the ambient tokio runtime and uses tokio timers. +/// Spawns onto one tokio runtime, captured as a handle at construction, and +/// uses that runtime's timers. +/// +/// The default instance is [`TokioRuntime::current`], taken while the cache is +/// built; [`DialCacheBuilder::build`](crate::DialCacheBuilder::build) fails +/// with a configuration error outside a tokio context instead of panicking on +/// first use. The runtime must have its time driver enabled (tokio's +/// `enable_time` or `enable_all`), or the first deadline panics inside the +/// detached work and surfaces as [`Error::Panic`](crate::Error::Panic). +#[cfg(feature = "tokio")] +#[derive(Debug, Clone)] +pub struct TokioRuntime { + handle: tokio::runtime::Handle, +} + #[cfg(feature = "tokio")] -#[derive(Debug, Clone, Copy, Default)] -pub struct TokioRuntime; +impl TokioRuntime { + /// The runtime of the current tokio context. + pub fn current() -> Result { + tokio::runtime::Handle::try_current() + .map(Self::from_handle) + .map_err(|_| { + crate::error::ConfigError::invalid( + "DialCache requires a tokio runtime: build the cache inside one, or \ + configure DialCacheBuilder::runtime with TokioRuntime::from_handle", + ) + }) + } + + /// Use the runtime behind `handle`, whichever context later calls the cache. + pub fn from_handle(handle: tokio::runtime::Handle) -> Self { + TokioRuntime { handle } + } +} #[cfg(feature = "tokio")] impl Runtime for TokioRuntime { fn spawn(&self, task: BoxFuture<'static, ()>) { - tokio::spawn(task); + self.handle.spawn(task); } fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { + // Enter the captured runtime so the timer binds to its time driver + // rather than to whichever runtime happens to call the cache. + let _enter = self.handle.enter(); Box::pin(tokio::time::sleep(duration)) } } diff --git a/rust/src/scope.rs b/rust/src/scope.rs index 86957c45..923cc0e0 100644 --- a/rust/src/scope.rs +++ b/rust/src/scope.rs @@ -44,10 +44,17 @@ impl Owner { } pub(crate) fn close(&self) { - let mut state = self.state.lock(); - state.live = false; - state.memo.clear(); - state.flights.clear(); + let retired = { + let mut state = self.state.lock(); + state.live = false; + ( + std::mem::take(&mut state.memo), + std::mem::take(&mut state.flights), + ) + }; + // Memoized values drop here, outside the lock: a value's destructor + // may call back into the cache. + drop(retired); } } diff --git a/rust/src/shadow.rs b/rust/src/shadow.rs index 7decd59d..c65cdd6d 100644 --- a/rust/src/shadow.rs +++ b/rust/src/shadow.rs @@ -11,6 +11,7 @@ use futures::FutureExt; use parking_lot::Mutex; use crate::deadline::{await_deadline, since}; +use crate::engine::Core; use crate::error::Error; use crate::execution::{Execution, RawRead}; use crate::flight::{panic_message, start_pending, yield_deferred, Settled, ValueResult}; @@ -23,6 +24,27 @@ use crate::preview::{preview_key, preview_value}; use crate::remote::{Frame, ReadResult}; /// One admitted shadow job holding an instance slot. +/// One held entry of the instance-wide shadow table. Dropping it frees the +/// slot, so a job dropped unpolled at runtime shutdown cannot pin capacity. +pub(crate) struct ShadowSlot { + core: Arc, + key: String, + flight: Arc, +} + +impl Drop for ShadowSlot { + fn drop(&mut self) { + let mut state = self.core.state.lock(); + if state + .shadows + .get(&self.key) + .is_some_and(|f| Arc::ptr_eq(f, &self.flight)) + { + state.shadows.remove(&self.key); + } + } +} + pub(crate) struct ShadowFlight { abandoned: AtomicBool, stop: Settled<()>, @@ -121,12 +143,17 @@ impl Execution { .insert(self.keys.logical.clone(), flight.clone()); flight }; + let slot = ShadowSlot { + core: self.core.clone(), + key: self.keys.logical.clone(), + flight: flight.clone(), + }; if p.logging_config_error { self.error_event(Layer::Remote, ErrorKind::ConfigResolution, false); } let x = self.clone(); self.core.runtime.defer(Box::pin(async move { - x.run_shadow(flight, frame, source, started).await + x.run_shadow(flight, frame, source, started, slot).await })); } @@ -172,6 +199,7 @@ impl Execution { frame: Option, source: Option>, started: Duration, + slot: ShadowSlot, ) { let clock = self.core.clock.clone(); let started = if source.is_none() { @@ -208,22 +236,15 @@ impl Execution { }; // Reads keep the slot after a read timeout; owned source, codec and // write work already keeps this operation running until raw completion. + // Dropping the slot frees it, whether the reads settled or the + // runtime dropped this task first. let pending = std::mem::take(&mut *reads.lock()); - let core = x.core.clone(); - let runtime = core.runtime.clone(); - let key = x.keys.logical.clone(); + let runtime = x.core.runtime.clone(); runtime.spawn(Box::pin(async move { for read in pending { let _ = read.wait().await; } - let mut state = core.state.lock(); - if state - .shadows - .get(&key) - .is_some_and(|f| Arc::ptr_eq(f, &flight)) - { - state.shadows.remove(&key); - } + drop(slot); })); verdict } diff --git a/rust/src/testing.rs b/rust/src/testing.rs index d6a2697f..e9aa4451 100644 --- a/rust/src/testing.rs +++ b/rust/src/testing.rs @@ -13,7 +13,7 @@ use std::sync::Arc; use std::task::{Context, Poll, Waker}; use std::time::Duration; -use crate::clock::Clock; +use crate::clock::{Clock, SystemClock}; use crate::runtime::Runtime; use futures::executor::{LocalPool, LocalSpawner}; use futures::future::BoxFuture; @@ -362,44 +362,21 @@ impl TestExecutor { } } -/// A clock aligned to the shared millisecond grid of a [`VirtualClock`], the -/// way [`SystemClock`](crate::SystemClock) aligns to the process grid. -/// Instances constructed at different fractional times share one grid. -pub struct VirtualGridClock { - base: Arc, - origin_ns: u128, -} - -impl std::fmt::Debug for VirtualGridClock { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("VirtualGridClock") - .field("origin_ns", &self.origin_ns) - .finish() - } -} - -impl VirtualGridClock { - pub fn new(base: Arc) -> Self { - let now_ns = base.state.lock().elapsed_ns; - VirtualGridClock { - base, - origin_ns: crate::clock::grid_origin_ns(now_ns), - } - } -} - -impl Clock for VirtualGridClock { - fn wall_ms(&self) -> i64 { - self.base.wall_ms() - } - - fn elapsed(&self) -> Duration { - let ns = self - .base - .state - .lock() - .elapsed_ns - .saturating_sub(self.origin_ns); - Duration::from_nanos(ns.min(u64::MAX as u128) as u64) - } +/// The production [`SystemClock`] built over a [`VirtualClock`]. +/// +/// Its origin aligns to the shared millisecond grid of the virtual elapsed +/// reading exactly as default instances align to the process grid, so +/// instances constructed at different fractional virtual times share one +/// expiry grid and the default alignment itself is what a controlled history +/// exercises. +pub fn grid_clock(base: &Arc) -> SystemClock { + let monotonic = base.clone(); + let wall = base.clone(); + SystemClock::with_sources( + move || { + let ns = monotonic.state.lock().elapsed_ns; + Duration::from_nanos(ns.min(u64::MAX as u128) as u64) + }, + move || wall.wall_ms(), + ) } diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index f51188b5..f4ff0ad5 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -168,8 +168,9 @@ impl Run { let full = self.selection.core.is_directory(); for path in self.selection.core_paths()? { let id = inventory::trace_case_id("core", &path, full); + let started = formal::report::now_ms(); let outcome = self.replay_core(&path); - self.record(&id, outcome)?; + self.record(&id, started, outcome)?; } Ok(()) } @@ -211,8 +212,9 @@ impl Run { let full = self.selection.effects.is_directory(); for path in self.selection.effects_paths()? { let id = inventory::trace_case_id("effects", &path, full); + let started = formal::report::now_ms(); let outcome = self.replay_behavior("effects", &path, true); - self.record(&id, outcome)?; + self.record(&id, started, outcome)?; } Ok(()) } @@ -238,8 +240,9 @@ impl Run { } for path in paths { let id = inventory::trace_case_id(&name, &path, full); + let started = formal::report::now_ms(); let outcome = self.replay_behavior(&name, &path, false); - self.record(&id, outcome)?; + self.record(&id, started, outcome)?; } } Ok(()) @@ -259,6 +262,7 @@ impl Run { let full = self.selection.features.is_directory(); for path in paths { let id = inventory::trace_case_id("local-clock", &path, full); + let started = formal::report::now_ms(); let outcome = (|| { let prepared = self.coordinator.prepare("local-clock", &path, None)?; note_actions(&mut self.seen_actions, "local-clock", &prepared); @@ -275,7 +279,7 @@ impl Run { &mut [], ) })(); - self.record(&id, outcome)?; + self.record(&id, started, outcome)?; } Ok(()) } @@ -297,8 +301,9 @@ impl Run { } matched += 1; let id = inventory::scenario_case_id(feature, name); + let started = formal::report::now_ms(); let outcome = scenarios::replay_scenario(scenario); - self.record(&id, outcome)?; + self.record(&id, started, outcome)?; } if matched == 0 { return Err("no behavioral scenario matched".to_string()); @@ -332,8 +337,9 @@ impl Run { for vector in vectors { let name = vector.get("name").and_then(Value::as_str).unwrap_or(""); let id = inventory::protocol_case_id(group, name); + let started = formal::report::now_ms(); let outcome = check(vector); - self.record(&id, outcome)?; + self.record(&id, started, outcome)?; } } Ok(()) @@ -361,15 +367,20 @@ impl Run { let root = repo_root(); for (profile, paths) in profiles { let id = inventory::witness_case_id(&profile); + let started = formal::report::now_ms(); let outcome = formal::witness::check_witness_evidence(&root, &profile, &directory, &paths); - self.record(&id, outcome)?; + self.record(&id, started, outcome)?; } Ok(()) } - fn record(&mut self, id: &str, outcome: Result<(), String>) -> Result<(), String> { - let started = formal::report::now_ms(); + fn record( + &mut self, + id: &str, + started: i64, + outcome: Result<(), String>, + ) -> Result<(), String> { if let Err(message) = &outcome { self.failures += 1; eprintln!("FAIL {id}\n{message}"); diff --git a/rust/tests/formal/driver.rs b/rust/tests/formal/driver.rs index ffebae96..e2b5c02e 100644 --- a/rust/tests/formal/driver.rs +++ b/rust/tests/formal/driver.rs @@ -12,8 +12,8 @@ use dialcache::protocol::{decode_frame, encode_frame, read_result_from_untrusted use dialcache::testing::{TestExecutor, VirtualClock}; use dialcache::{ BoxError, Codec, DialCache, Error, FallbackTimeout, Identity, InvalidateRequest, LocalEntry, - LocalStore, LruLocalStore, Operation, Payload, Policy, ReadContext, ReadRequest, ReadResult, - Remote, RuntimePolicy, Scope, SourceBudget, StoredValue, WriteRequest, + LocalRead, LocalStore, LruLocalStore, Operation, Payload, Policy, ReadContext, ReadRequest, + ReadResult, Remote, RuntimePolicy, Scope, SourceBudget, WriteRequest, }; use futures::future::BoxFuture; use parking_lot::Mutex; @@ -1501,23 +1501,23 @@ struct FaultStore { } impl LocalStore for FaultStore { - fn get(&mut self, key: &str, now_ms: i64) -> Result, BoxError> { + fn get(&mut self, key: &str, now_ms: i64) -> Result { if self.shared.lock().fault("localStorage") { return Err("controlled local storage failure".into()); } match self.inner.as_mut() { Some(inner) => inner.get(key, now_ms), - None => Ok(None), + None => Ok(LocalRead::Absent), } } - fn put(&mut self, key: String, entry: LocalEntry) -> Result<(), BoxError> { + fn put(&mut self, key: String, entry: LocalEntry) -> Result, BoxError> { if self.shared.lock().fault("localStorage") { return Err("controlled local storage failure".into()); } match self.inner.as_mut() { Some(inner) => inner.put(key, entry), - None => Ok(()), + None => Ok(None), } } } diff --git a/rust/tests/formal/local_clock.rs b/rust/tests/formal/local_clock.rs index 63e853c6..97d6b6ba 100644 --- a/rust/tests/formal/local_clock.rs +++ b/rust/tests/formal/local_clock.rs @@ -1,9 +1,10 @@ -//! The local-clock profile driver: default-aligned instances over a shared -//! virtual grid, fractional environment ticks, immediate healthy sources. +//! The local-clock profile driver: default instances whose production clock +//! alignment runs over a shared virtual grid, fractional environment ticks, +//! immediate healthy sources. use std::sync::Arc; -use dialcache::testing::{TestExecutor, VirtualGridClock}; +use dialcache::testing::{grid_clock, TestExecutor}; use dialcache::{DialCache, Identity, Operation, Policy}; use parking_lot::Mutex; use serde_json::{json, Map, Value}; @@ -41,7 +42,7 @@ impl LocalClockDriver { return Err("invalid/duplicate instance".to_string()); } let cache = DialCache::builder() - .clock(VirtualGridClock::new(self.exec.clock.clone())) + .clock(grid_clock(&self.exec.clock)) .runtime_arc(self.exec.runtime.clone()) .build() .map_err(|e| e.to_string())?; diff --git a/rust/tests/metrics_exporters.rs b/rust/tests/metrics_exporters.rs index 18038288..71346db7 100644 --- a/rust/tests/metrics_exporters.rs +++ b/rust/tests/metrics_exporters.rs @@ -628,12 +628,13 @@ mod prometheus_exporter { } } - /// Port of TestPrometheusReuseAndConflictIsolation. + /// Port of TestPrometheusReuseAndConflictIsolation. Rust shares one + /// observer by cloning it rather than by re-registering the same names. #[test] fn reuse_and_conflict_isolation() { let registry = Registry::new(); let first = PrometheusObserver::new(®istry, "test_").expect("first"); - let second = PrometheusObserver::new(®istry, "test_").expect("second reuses"); + let second = first.clone(); first.observe(&metric_test_event(MetricKind::Request)); second.observe(&metric_test_event(MetricKind::Request)); first.observe(&metric_test_event(MetricKind::Get)); @@ -660,10 +661,10 @@ mod prometheus_exporter { } assert!(saw_counter && saw_histogram, "metrics not exported"); - // Reuse survives dropping every earlier observer. + // Clones survive dropping the original observer. + let third = second.clone(); drop(first); drop(second); - let third = PrometheusObserver::new(®istry, "test_").expect("third reuses"); third.observe(&metric_test_event(MetricKind::Request)); let requests = registry .gather() @@ -672,6 +673,24 @@ mod prometheus_exporter { .expect("request family"); assert_eq!(requests.get_metric()[0].get_counter().value(), 3.0); + // A second registration of the same names is a conflict that leaves + // the registry, and the shared series, untouched. + let duplicate = PrometheusObserver::new(®istry, "test_") + .expect_err("re-registered the same collectors"); + match &duplicate { + PrometheusError::Conflict { name, .. } => { + assert_eq!(name, "test_dialcache_disabled_counter") + } + other => panic!("unexpected error {other}"), + } + third.observe(&metric_test_event(MetricKind::Request)); + let requests = registry + .gather() + .into_iter() + .find(|f| f.name() == "test_dialcache_request_counter") + .expect("request family"); + assert_eq!(requests.get_metric()[0].get_counter().value(), 4.0); + // Another prefix on the same registry is an independent group. let other = PrometheusObserver::new(®istry, "other_").expect("other prefix"); other.observe(&metric_test_event(MetricKind::Request)); diff --git a/rust/tests/settlement_control.rs b/rust/tests/settlement_control.rs index 7ff83cf3..4f5e62c2 100644 --- a/rust/tests/settlement_control.rs +++ b/rust/tests/settlement_control.rs @@ -10,7 +10,7 @@ use formal::inventory::repo_path; use formal::transport::Coordinator; use serde_json::Value; -const BEHAVIOR_SMOKE: [&str; 14] = [ +const BEHAVIOR_SMOKE: [&str; 13] = [ "effects", "admission", "independent", @@ -24,7 +24,6 @@ const BEHAVIOR_SMOKE: [&str; 14] = [ "shadow", "shadow-layers", "source-budgets", - "effects", ]; fn replay(coordinator: &mut Coordinator, profile: &str, skip_settle: bool) -> Result<(), String> { @@ -54,9 +53,7 @@ fn replay(coordinator: &mut Coordinator, profile: &str, skip_settle: bool) -> Re fn unsettled_observations_fail_every_behavior_smoke_history() { install_panic_hook(); let mut coordinator = Coordinator::spawn().expect("coordinator"); - let mut profiles: Vec<&str> = BEHAVIOR_SMOKE.to_vec(); - profiles.dedup(); - for profile in profiles { + for profile in BEHAVIOR_SMOKE { replay(&mut coordinator, profile, false) .unwrap_or_else(|e| panic!("{profile} settled replay must pass: {e}")); let unsettled = replay(&mut coordinator, profile, true); diff --git a/rust/tests/tokio_runtime.rs b/rust/tests/tokio_runtime.rs new file mode 100644 index 00000000..f31574dd --- /dev/null +++ b/rust/tests/tokio_runtime.rs @@ -0,0 +1,168 @@ +//! The production tokio path: the default runtime captured at build time, +//! coalescing under a multi-thread scheduler, source deadlines on tokio +//! timers, and the cancellation contracts the deterministic harness cannot +//! exercise (dropped `enable` futures, dropped callers). + +#![cfg(feature = "tokio")] + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use dialcache::{DialCache, Identity, Operation, Policy, Scope, SourceBudget, TokioRuntime}; + +fn operation(id: &str) -> Operation { + Operation::::new(Identity::new("thing", id, "TokioSmoke")) + .policy(Policy::default().local_ttl_sec(60)) +} + +fn source( + calls: &Arc, + delay: Duration, + value: u64, +) -> impl Fn(Scope) -> futures::future::BoxFuture<'static, Result> + + Send + + Sync + + 'static { + let calls = calls.clone(); + move |_| { + let calls = calls.clone(); + Box::pin(async move { + tokio::time::sleep(delay).await; + calls.fetch_add(1, Ordering::SeqCst); + Ok(value) + }) + } +} + +#[test] +fn building_outside_a_tokio_context_is_a_configuration_error() { + let error = match DialCache::builder().build() { + Err(error) => error, + Ok(_) => panic!("built a default runtime without a tokio context"), + }; + assert!( + error.to_string().contains("tokio runtime"), + "unexpected error: {error}" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn concurrent_callers_coalesce_and_later_callers_hit() { + let cache = DialCache::builder().build().expect("built inside tokio"); + let calls = Arc::new(AtomicUsize::new(0)); + let request = cache.enable_guard(); + let mut pending = Vec::new(); + for _ in 0..8 { + let cache = cache.clone(); + let scope = request.scope().clone(); + let load = source(&calls, Duration::from_millis(20), 7); + pending.push(tokio::spawn(async move { + cache.get_or_load(&scope, operation("one"), load).await + })); + } + for handle in pending { + let value = handle.await.expect("task").expect("value"); + assert_eq!(*value, 7); + } + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "callers were not coalesced" + ); + let again = cache + .get_or_load( + request.scope(), + operation("one"), + source(&calls, Duration::ZERO, 8), + ) + .await + .expect("hit"); + assert_eq!(*again, 7); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "a local hit called the source" + ); +} + +#[tokio::test] +async fn a_source_deadline_returns_a_timeout_and_leaves_the_source_running() { + let cache = DialCache::builder() + .runtime(TokioRuntime::from_handle(tokio::runtime::Handle::current())) + .build() + .expect("explicit handle"); + let calls = Arc::new(AtomicUsize::new(0)); + let request = cache.enable_guard(); + let error = cache + .get_or_load( + request.scope(), + operation("slow").budget(SourceBudget::Millis(20)), + source(&calls, Duration::from_millis(150), 1), + ) + .await + .expect_err("deadline"); + assert!(error.is_fallback_timeout(), "unexpected error: {error}"); + assert_eq!(calls.load(Ordering::SeqCst), 0); + tokio::time::sleep(Duration::from_millis(300)).await; + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "the deadline cancelled the source" + ); +} + +#[tokio::test] +async fn dropping_the_enable_future_closes_its_scope() { + let cache = DialCache::builder().build().expect("built inside tokio"); + let (send, receive) = tokio::sync::oneshot::channel::(); + let inner = cache.clone(); + let outcome = tokio::time::timeout( + Duration::from_millis(10), + cache.enable(|scope| async move { + assert!(inner.is_enabled(&scope)); + let _ = send.send(scope); + std::future::pending::<()>().await; + }), + ) + .await; + assert!(outcome.is_err(), "the callback completed"); + let retained = receive.await.expect("scope handed out"); + assert!( + !cache.is_enabled(&retained), + "a dropped enable future left its scope live" + ); +} + +#[tokio::test] +async fn dropping_the_caller_future_does_not_cancel_the_execution() { + let cache = DialCache::builder().build().expect("built inside tokio"); + let calls = Arc::new(AtomicUsize::new(0)); + let request = cache.enable_guard(); + let abandoned = tokio::time::timeout( + Duration::from_millis(5), + cache.get_or_load( + request.scope(), + operation("kept"), + source(&calls, Duration::from_millis(50), 3), + ), + ) + .await; + assert!(abandoned.is_err(), "the source finished within 5 ms"); + tokio::time::sleep(Duration::from_millis(150)).await; + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "the source did not complete" + ); + let value = cache + .get_or_load( + request.scope(), + operation("kept"), + source(&calls, Duration::ZERO, 4), + ) + .await + .expect("published value"); + assert_eq!(*value, 3, "the abandoned execution did not publish"); + assert_eq!(calls.load(Ordering::SeqCst), 1); +} diff --git a/test/formal-rust-replay.test.ts b/test/formal-rust-replay.test.ts index f733ad39..091075c7 100644 --- a/test/formal-rust-replay.test.ts +++ b/test/formal-rust-replay.test.ts @@ -119,13 +119,16 @@ describe("Rust replay report gate", () => { describe("Rust validation lanes", () => { const directory = "/checkout"; - it("checks formatting, clippy and default tests from the repository root through the crate manifest", () => { + it("checks formatting, clippy and default tests from inside the crate so rustup honors its toolchain pin", () => { expect(validationPlan("check-rust", { directory }).map(step => [step.command, ...step.args!])).toEqual([ - ["cargo", "fmt", "--manifest-path", "rust/Cargo.toml", "--check"], - ["cargo", "clippy", "--manifest-path", "rust/Cargo.toml", "--all-targets", "--all-features", "--", "-D", "warnings"], - ["cargo", "test", "--manifest-path", "rust/Cargo.toml", "--all-features"], + ["cargo", "fmt", "--check"], + ["cargo", "clippy", "--all-targets", "--all-features", "--", "-D", "warnings"], + ["cargo", "test", "--all-features"], ]); - for (const step of validationPlan("check-rust", { directory })) expect(step.env, step.label).toBeUndefined(); + for (const step of validationPlan("check-rust", { directory })) { + expect(step.env, step.label).toBeUndefined(); + expect(step.cwd, step.label).toBe("rust"); + } expect(validationPlan("check", { directory })).toEqual(["check-ts", "check-go", "check-rust", "docs", "audit"].flatMap(target => validationPlan(target, { directory }))); }); @@ -134,7 +137,8 @@ describe("Rust validation lanes", () => { expect(plan[0]).toEqual({ label: "Invalidate prior rust completion", remove: [".formal-traces/rust-completion.json"] }); expect(plan[1]!.args).toEqual(["formal/conformance.mjs", "prepare", "rust", ".formal-traces/rust-context.json"]); const replay = plan[2]!; - expect([replay.command, ...replay.args!]).toEqual(["cargo", "test", "--manifest-path", "rust/Cargo.toml", "--release", "--all-features", "--test", "conformance"]); + expect([replay.command, ...replay.args!]).toEqual(["cargo", "test", "--release", "--all-features", "--test", "conformance"]); + expect(replay.cwd).toBe("rust"); expect(replay.env).toEqual({ DIALCACHE_MBT_TRACE_DIR: "/checkout/.formal-traces/conformance", DIALCACHE_EFFECTS_TRACE_DIR: "/checkout/.formal-traces/effects", @@ -157,17 +161,24 @@ describe("Rust validation lanes", () => { it("adds the smoke conformance run in default mode with no corpus selectors", () => { const smoke = validationPlan("smoke", { directory }); - expect(smoke.at(-1)).toEqual({ label: "Replay committed Rust fixtures", command: "cargo", args: ["test", "--manifest-path", "rust/Cargo.toml", "--all-features", "--test", "conformance"] }); + expect(smoke.at(-1)).toEqual({ label: "Replay committed Rust fixtures", command: "cargo", args: ["test", "--all-features", "--test", "conformance"], cwd: "rust" }); expect(smoke.filter(step => step.command === "cargo")).toHaveLength(1); }); + it("runs the real-server integration binary only through the integration lane, opted in by its environment variable", () => { + const lane = validationPlan("integration-rust", { directory }); + expect(lane).toEqual([{ label: "Run Rust Redis/Valkey/Cluster integrations", command: "cargo", args: ["test", "--all-features", "--test", "redis_integration"], cwd: "rust", env: { DIALCACHE_RUST_INTEGRATION: "1" } }]); + expect(validationPlan("integration", { directory })).toEqual(["integration-ts", "integration-go", "integration-rust"].flatMap(target => validationPlan(target, { directory }))); + for (const target of ["check-rust", "smoke", "formal-rust"]) expect(validationPlan(target, { directory }).some(step => step.env?.DIALCACHE_RUST_INTEGRATION), target).toBe(false); + }); + it("probes the pinned cargo exactly for the Rust lanes", async () => { const { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } = await import("node:fs"); const { tmpdir } = await import("node:os"); const { delimiter, join } = await import("node:path"); const temporary = mkdtempSync(join(tmpdir(), "dialcache-rust-prereq-")); try { - for (const path of ["bin", "formal", "node_modules/typescript"]) mkdirSync(join(temporary, path), { recursive: true }); + for (const path of ["bin", "formal", "node_modules/typescript", "rust"]) mkdirSync(join(temporary, path), { recursive: true }); writeFileSync(join(temporary, "package.json"), '{"packageManager":"pnpm@10.33.0"}'); writeFileSync(join(temporary, "node_modules/typescript/package.json"), "{}"); writeFileSync(join(temporary, "formal/generated-fixtures.lock.json"), '{"quintVersion":"0.32.0"}'); @@ -178,7 +189,7 @@ describe("Rust validation lanes", () => { const environment = { ...process.env, PATH: `${join(temporary, "bin")}${delimiter}${process.env.PATH ?? ""}` }; const options = { directory: temporary, environment, nodeVersion: "v24.20.0" }; tool("cargo", 'console.error("cargo: command not found"); process.exit(127)'); - for (const target of ["check-rust", "formal-rust", "smoke", "check"]) expect(() => checkPrerequisites(target, options), target).toThrow(/Cannot run cargo/); + for (const target of ["check-rust", "formal-rust", "smoke", "check", "integration-rust"]) expect(() => checkPrerequisites(target, options), target).toThrow(/Cannot run cargo/); for (const target of ["check-ts", "check-go", "formal-ts", "formal-go", "mutations", "audit"]) expect(() => checkPrerequisites(target, options), target).not.toThrow(); tool("cargo", 'console.log("cargo 1.97.0 (abcdef 2026-06-01)")'); expect(() => checkPrerequisites("check-rust", options)).toThrow(/requires cargo 1\.98\.1; found cargo 1\.97\.0/); diff --git a/test/formal-validation.test.ts b/test/formal-validation.test.ts index fad5b4e1..46fe791f 100644 --- a/test/formal-validation.test.ts +++ b/test/formal-validation.test.ts @@ -45,7 +45,7 @@ describe("shared validation runner", () => { beforeEach(() => { directory = mkdtempSync(join(tmpdir(), "dialcache-validation-")); - for (const path of ["bin", "formal", "dist", "node_modules/typescript"]) mkdirSync(join(directory, path), { recursive: true }); + for (const path of ["bin", "formal", "dist", "node_modules/typescript", "rust"]) mkdirSync(join(directory, path), { recursive: true }); child = join(directory, "child.mjs"); put("child.mjs", `import { appendFileSync } from 'node:fs'; appendFileSync(process.env.RUNNER_EVENTS, JSON.stringify({ label: process.argv[2], cwd: process.cwd(), @@ -87,6 +87,16 @@ process.exit(Number(process.argv[3] ?? 0));\n`); }); }); + it("runs a step inside its declared directory and every other step at the checkout root", async () => { + await run([ + { label: "root", command: process.execPath, args: [child, "root"] }, + { label: "crate", command: process.execPath, args: [child, "crate"], cwd: "rust" }, + ]); + const [root, crate] = events(); + expect(root!.cwd).toBe(realpathSync(directory)); + expect(crate!.cwd).toBe(realpathSync(join(directory, "rust"))); + }); + it("stops at a failing child and preserves its partial native report without running later steps", async () => { const steps: Step[] = [ { label: "first", command: process.execPath, args: [child, "first"] }, From ea83f6efc528683e1e47dd885989fc2c3bd17cd3 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 01:40:52 -0700 Subject: [PATCH 19/31] chore(formal): refresh the guide audit pins for the three-port wording --- formal/go-parity.json | 4 ++-- formal/source-audit.json | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/formal/go-parity.json b/formal/go-parity.json index 37e43d12..cf65a783 100644 --- a/formal/go-parity.json +++ b/formal/go-parity.json @@ -10,7 +10,7 @@ "inputs": { "semanticCasesSha256": "7054107347f57a452e17de99f74310553b7af4350363c674e70ff8184d0610be", "executionSha256": "feee09a3487deb2694f8784b7e8d2d0ee28050dae03ac43ac2613608ce5ab1c6", - "sourceAuditSha256": "ca0fcbb9b190ccfcb101d427d61794abf62296d1618c28156e8301bbb52fe0d1", + "sourceAuditSha256": "9a877f15ce14c557e52c3c0e11527cef4b56923b8ff472e595834f88d4bd9f29", "quintCaseAuditSha256": "dcb576948a68a066e43259d4bee2b3cdadbe82e657ae9af99e4b21ed7b95dbb3", "featureCoverageSha256": "56eedfa865682db6ad8e73b8673c7dac1c30d301e9bce3390246d6994aba9dcb", "coverageWitnessesSha256": "8fc8ab123150fdec059f5b75f84172d39af80d08d7a4211fa88c63c5a3f2874b", @@ -18305,7 +18305,7 @@ ], "reviewedTestAndDocumentationAudit": { "path": "formal/source-audit.json", - "sha256": "ca0fcbb9b190ccfcb101d427d61794abf62296d1618c28156e8301bbb52fe0d1", + "sha256": "9a877f15ce14c557e52c3c0e11527cef4b56923b8ff472e595834f88d4bd9f29", "sourceFiles": 44, "status": "reviewed-applicability-not-execution", "limitations": "These rows review Go applicability for the public documentation and ordinary test files in source-audit.json sources, inheriting their C/W/E/B/X classifications. They map subjects to native artifacts without claiming one Go test per TypeScript assertion or turning documentation counts into executed coverage. The same audit hash also binds reviewedGuides for formal and Go documentation; those guide reviews are freshness/scope records, not additional behavioral applicability rows. Per-case Quint and replay evidence is recorded separately.", diff --git a/formal/source-audit.json b/formal/source-audit.json index 755db4ad..9e797660 100644 --- a/formal/source-audit.json +++ b/formal/source-audit.json @@ -1287,7 +1287,7 @@ }, { "path": "formal/PORTING.md", - "sha256": "9280203fd477f26f728e8334a8541a8755f0977c32913b7db08cea0256077c25", + "sha256": "204d3c424f89c3014641e1027d464bacc7c6099ef3ee47ea373b30db92062841", "entries": [ {"line":1,"title":"Implementing another DialCache port"}, {"line":14,"title":"What a port supplies"}, @@ -1296,9 +1296,9 @@ {"line":146,"title":"Observation contract"}, {"line":222,"title":"Generated fixtures and fast local tests"}, {"line":257,"title":"Complete corpus and reusable acceptance checks"}, - {"line":365,"title":"Witness evidence"}, - {"line":455,"title":"Current limitations for a further port"}, - {"line":469,"title":"New-port acceptance"} + {"line":370,"title":"Witness evidence"}, + {"line":460,"title":"Current limitations for a further port"}, + {"line":474,"title":"New-port acceptance"} ], "review": { "kind": "tooling-guide", @@ -1346,14 +1346,14 @@ }, { "path": "formal/README.md", - "sha256": "408c23982e53874489d1aa804c7ece96b7f487cf7e0a4a07552fcf914ca015d7", + "sha256": "d612ebd99ef15cfe0540524163e5fb904bb416171909aa20b5df24894d3f242c", "entries": [ {"line":1,"title":"Executable DialCache specification"}, {"line":7,"title":"Start with your task"}, {"line":23,"title":"How the specification connects to code"}, {"line":43,"title":"Models and composition profiles"}, {"line":94,"title":"Generating and replaying behavior"}, - {"line":181,"title":"Evidence and scope"} + {"line":182,"title":"Evidence and scope"} ], "review": { "kind": "tooling-guide", @@ -1533,7 +1533,7 @@ }, { "path": "go/README.md", - "sha256": "8527fa4575ed1739abf134def472decac898060bb69f3e9b9ced4dfbe6e1c26d", + "sha256": "c859de57f5cb4c7c785f0510f009ef3a5f71154a0facf773fe9c2bebc7959c6a", "entries": [ {"line":1,"title":"DialCache for Go"}, {"line":19,"title":"Use"}, From 25ef575254fc32136023efe4864d5594c9d75508 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 01:44:06 -0700 Subject: [PATCH 20/31] test(formal): type the per-step working directory in the validation lane tests --- test/formal-rust-replay.test.ts | 2 +- test/formal-validation.test.ts | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test/formal-rust-replay.test.ts b/test/formal-rust-replay.test.ts index 091075c7..277f3fe2 100644 --- a/test/formal-rust-replay.test.ts +++ b/test/formal-rust-replay.test.ts @@ -6,7 +6,7 @@ type Summary = { quintRegressions: Record; quintRegressionTraces: number; fixedScenarios: number; protocolVectors: number; witnessProfiles: string[]; executedCases: number; }; -type Step = { label: string; command?: string; args?: string[]; env?: NodeJS.ProcessEnv; stdoutFile?: string; remove?: string[] }; +type Step = { label: string; command?: string; args?: string[]; cwd?: string; env?: NodeJS.ProcessEnv; stdoutFile?: string; remove?: string[] }; type Record_ = Record; const { checkRustReplay } = await import(new URL("../formal/check-rust-replay.mjs", import.meta.url).href) as { diff --git a/test/formal-validation.test.ts b/test/formal-validation.test.ts index 46fe791f..c9d7c69c 100644 --- a/test/formal-validation.test.ts +++ b/test/formal-validation.test.ts @@ -9,6 +9,7 @@ type Step = { label: string; command?: string; args?: string[]; + cwd?: string; env?: NodeJS.ProcessEnv; stdoutFile?: string; requireEmptyStdout?: boolean; From 4d1c715f4ed80d017333a1913efb8d1145e255c9 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sat, 19 Sep 2026 12:13:32 -0700 Subject: [PATCH 21/31] feat(formal): Rust mutation lane, cluster primary-routing test and second-round review fixes Mutation lane. make mutations-rust / make mutations-merge-rust measure the Rust fault catalog formal/rust-mutations.json (13 single-site faults with the same contract cases as the TypeScript and Go catalogs) through formal/measure-rust-semantics.mjs: each mutant is applied to an isolated copy of the crate, compiled in release mode with dependencies shared through rust/target/semantic, and run against three cohorts. Ordinary is the crate's unit and native tests; generated and fixed are the conformance harness under the new DIALCACHE_RUST_SUITE selector over the complete corpus or the fixed scenarios. The lane shares the shard partition, fingerprint, gate and merge with the other ports (languages.rust; fingerprints now accept file paths and exclude the build directory), runs as three shards plus a merge in the full formal workflow, and is required by the aggregate. First complete run: 13/13 detected by the generated cohort, 12/13 by the fixed scenarios (M09 survives fixed, as in Go), 5/13 by ordinary tests, 778 s unsharded. Cluster routing. A Docker-backed test builds a Redis 7 primary plus replica on a private network, remaps their announced addresses with node_address_map, enables replica reads, and asserts from INFO commandstats that every tracked MGET executed on the primary and none on the replica; the mutation to ReplicaOptional fails it. The Docker tests are #[ignore]d so a plain cargo test reports them as ignored, and make integration-rust runs them with --ignored. Review round two (19 confirmed findings): a displaced request-memo value now drops outside the owner lock; the Prometheus rollback docs state what the prometheus crate keeps; the exporter tests assert each kind's full tag list instead of a vacuous never-exported check; the scripted Redis double fails loudly on an unexpected dispatch and hangs only on request; the smoke and exploration workflows install the pinned Rust toolchain; explore probes cargo; an incomplete Rust exploration report is diagnosed as such; the tokio deadline test releases its source explicitly instead of racing timers; and the rustdoc for Request, Fallback, Warn, config_error, build and the compression read result now matches the code. --- .github/workflows/formal-full.yaml | 70 ++- .github/workflows/formal.yaml | 3 +- Makefile | 4 +- formal/PORTING.md | 2 +- formal/README.md | 10 +- formal/SEMANTIC-COVERAGE.md | 33 +- formal/VALIDATION.md | 9 +- formal/explore.mjs | 3 +- formal/go-parity.json | 4 +- formal/measure-rust-semantics.mjs | 256 +++++++++++ formal/merge-mutation-reports.mjs | 6 +- formal/mutation-reports.mjs | 24 +- formal/rust-mutations.json | 302 +++++++++++++ formal/source-audit.json | 16 +- formal/validation.mjs | 23 +- rust/README.md | 12 +- rust/src/engine.rs | 4 +- rust/src/execution.rs | 14 +- rust/src/observe.rs | 20 +- rust/src/policy.rs | 4 +- rust/src/prometheus.rs | 9 +- rust/src/protocol/envelope.rs | 8 +- rust/src/redis.rs | 26 +- rust/tests/conformance.rs | 25 +- rust/tests/formal/inventory.rs | 35 ++ rust/tests/metrics_exporters.rs | 14 +- rust/tests/redis_integration.rs | 531 ++++++++++++++++++++--- rust/tests/tokio_runtime.rs | 29 +- test/formal-mutation-shards.test.ts | 49 ++- test/formal-rust-replay.test.ts | 10 +- test/formal-rust-semantic-runner.test.ts | 123 ++++++ test/formal-validation.test.ts | 11 +- 32 files changed, 1520 insertions(+), 169 deletions(-) create mode 100644 formal/measure-rust-semantics.mjs create mode 100644 formal/rust-mutations.json create mode 100644 test/formal-rust-semantic-runner.test.ts diff --git a/.github/workflows/formal-full.yaml b/.github/workflows/formal-full.yaml index 78d44dd2..648dad2c 100644 --- a/.github/workflows/formal-full.yaml +++ b/.github/workflows/formal-full.yaml @@ -31,7 +31,8 @@ jobs: with: quint: "true" go: "true" - - name: Explore a fresh recorded seed and replay both ports + rust: "true" + - name: Explore a fresh recorded seed and replay every port run: make explore - name: Preserve exploratory sources, seed and counterexamples if: always() @@ -343,11 +344,72 @@ jobs: if-no-files-found: warn retention-days: 14 + # The Rust lane compiles the crate and its test binaries in release mode + # once per mutant (dependencies are cached across mutants) and replays the + # generated cohort in about a minute; the budget mirrors go-mutations. + rust-mutations: + needs: generate + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shard: [1, 2, 3] + env: + MUTATION_SHARD: ${{ matrix.shard }}/3 + timeout-minutes: 45 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: ./.github/actions/setup-validation + with: + rust: "true" + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + name: formal-traces + path: .formal-traces + - name: Measure one shard of Rust assertion-based fault detection + run: make mutations-rust + - name: Preserve this shard's Rust mutation evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: rust-semantic-shard-${{ matrix.shard }} + path: .formal-traces/rust-semantic/shards/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + + rust-mutations-merge: + needs: rust-mutations + # Run after the shards regardless of their result so a missing shard is + # diagnosed by the merge; skip only when the matrix itself never ran. + if: always() && needs.rust-mutations.result != 'skipped' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: ./.github/actions/setup-validation + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 + with: + pattern: rust-semantic-shard-* + path: .formal-traces/rust-semantic/shards + merge-multiple: true + - name: Require every Rust shard and merge the complete report + run: make mutations-merge-rust + - name: Preserve Rust mutation evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: rust-semantic-evidence + path: .formal-traces/rust-semantic/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 + # A failed or skipped dependency cannot turn smoke evidence into full acceptance. # The mutation lanes count through their merge jobs: a merge succeeds only # when every shard's report is present, consistent and free of lost detections. formal-full: - needs: [check-models, generate, typescript-parity, symbolic, typescript-mutations-merge, go-parity, go-mutations-merge, rust-parity, exploration] + needs: [check-models, generate, typescript-parity, symbolic, typescript-mutations-merge, go-parity, go-mutations-merge, rust-parity, rust-mutations-merge, exploration] if: always() runs-on: ubuntu-latest timeout-minutes: 5 @@ -418,7 +480,7 @@ jobs: include-hidden-files: true if-no-files-found: warn retention-days: 90 - - name: Require the model check, the generated corpus, every port replay and both merged mutation measurements + - name: Require the model check, the generated corpus, every port replay and every merged mutation measurement env: CHECK_MODELS_RESULT: ${{ needs.check-models.result }} GENERATE_RESULT: ${{ needs.generate.result }} @@ -426,6 +488,7 @@ jobs: TYPESCRIPT_MUTATIONS_MERGE_RESULT: ${{ needs.typescript-mutations-merge.result }} GO_RESULT: ${{ needs.go-parity.result }} GO_MUTATIONS_MERGE_RESULT: ${{ needs.go-mutations-merge.result }} + RUST_MUTATIONS_MERGE_RESULT: ${{ needs.rust-mutations-merge.result }} RUST_RESULT: ${{ needs.rust-parity.result }} SYMBOLIC_RESULT: ${{ needs.symbolic.result }} EXPLORATION_RESULT: ${{ needs.exploration.result }} @@ -437,6 +500,7 @@ jobs: test "$TYPESCRIPT_MUTATIONS_MERGE_RESULT" = success test "$GO_RESULT" = success test "$GO_MUTATIONS_MERGE_RESULT" = success + test "$RUST_MUTATIONS_MERGE_RESULT" = success test "$RUST_RESULT" = success test "$SYMBOLIC_RESULT" = success if [ "$EXPLORATION_REQUIRED" = true ]; then diff --git a/.github/workflows/formal.yaml b/.github/workflows/formal.yaml index 7eb0253a..2240fee8 100644 --- a/.github/workflows/formal.yaml +++ b/.github/workflows/formal.yaml @@ -24,7 +24,8 @@ jobs: - uses: ./.github/actions/setup-validation with: go: "true" - - name: Audit contracts and replay committed Quint smoke in both ports + rust: "true" + - name: Audit contracts and replay committed Quint smoke in every port run: make audit smoke - name: Detect changes requiring fresh Quint artifact generation id: fixture-scope diff --git a/Makefile b/Makefile index 2061e74f..76097672 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ NODE ?= node -.PHONY: help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go integration-rust package-floor ci explore model-check +.PHONY: help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go mutations-rust mutations-merge-rust integration integration-ts integration-go integration-rust package-floor ci explore model-check -help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go integration integration-ts integration-go integration-rust package-floor ci explore model-check: +help check check-ts check-go check-rust docs audit smoke formal formal-check formal-generate formal-ts formal-go formal-rust fixtures-check kernel-fixtures differential mutations mutations-ts mutations-go mutations-merge-ts mutations-merge-go mutations-rust mutations-merge-rust integration integration-ts integration-go integration-rust package-floor ci explore model-check: $(NODE) formal/validation.mjs $@ diff --git a/formal/PORTING.md b/formal/PORTING.md index 41355cdd..d5d04275 100644 --- a/formal/PORTING.md +++ b/formal/PORTING.md @@ -263,7 +263,7 @@ from the repository root: make check # Fast native checks and committed smoke; not full acceptance. make formal # Quint model/corpus checks, then prepared TS, Go and Rust replay. make model-check # Separate finite symbolic checks; requires Java 21 and tar. -make mutations # Measures both fault catalogs over the generated corpus. +make mutations # Measures every fault catalog over the generated corpus. make integration # Real-server interoperability; requires Docker. ``` diff --git a/formal/README.md b/formal/README.md index e70ada5d..d4371137 100644 --- a/formal/README.md +++ b/formal/README.md @@ -130,11 +130,11 @@ scheduled model with the Rust evaluator, the public regressions and the model mutation challenges. `make formal-generate` runs generation, fixture recomputation and the shared witness evaluation; `make formal-ts`, `make formal-go` and `make formal-rust` then complete each port's replay -against that exact corpus. `make mutations-ts` and `make mutations-go` split -the fault campaigns (the Rust port has no mutation lane yet). The parity and -mutation lanes depend only on the generated corpus and shared witness evidence, -so hosted CI runs all five in parallel and none of them waits for the model -check, which runs beside generation; the aggregate requires every lane. +against that exact corpus. `make mutations-ts`, `make mutations-go` and +`make mutations-rust` split the fault campaigns. The parity and mutation lanes +depend only on the generated corpus and shared witness evidence, so hosted CI +runs all six in parallel and none of them waits for the model check, which +runs beside generation; the aggregate requires every lane. `make fixtures-check` recomputes committed artifacts; after an intentional model edit, update them with `node formal/generate-artifacts.mjs --write` first. `make ci` includes the separate symbolic checks after `make formal`, as well as diff --git a/formal/SEMANTIC-COVERAGE.md b/formal/SEMANTIC-COVERAGE.md index 9b8cd95e..2060a4d2 100644 --- a/formal/SEMANTIC-COVERAGE.md +++ b/formal/SEMANTIC-COVERAGE.md @@ -133,23 +133,36 @@ Use the [shared Make targets and pinned prerequisites](./README.md#generating-an ```sh make formal # Regenerate and complete prepared TS and Go replay. -make mutations # Measure both fault catalogs over the generated corpus. +make mutations # Measure every fault catalog over the generated corpus. ``` -`make mutations-ts` and `make mutations-go` depend only on the corpus produced -by `make formal-generate` and its shared witness evidence, not on either port's -completion report. Hosted CI runs both mutation lanes in parallel with both -parity lanes off one generation job; the aggregate requires all of them. Each -report records the source, corpus and witness fingerprints it measured. Raw -measurement programs remain `measure-semantics.mjs` and -`measure-go-semantics.mjs`; the Make targets supply the pinned prerequisite +`make mutations-ts`, `make mutations-go` and `make mutations-rust` depend only +on the corpus produced by `make formal-generate` and its shared witness +evidence, not on any port's completion report. Hosted CI runs the three +mutation lanes in parallel with the three parity lanes off one generation job; +the aggregate requires all of them. Each report records the source, corpus and +witness fingerprints it measured. Raw measurement programs remain +`measure-semantics.mjs`, `measure-go-semantics.mjs` and +`measure-rust-semantics.mjs`; the Make targets supply the pinned prerequisite checks used by hosted full validation. +The Rust catalog [`rust-mutations.json`](./rust-mutations.json) names the same +13 contract cases. Its ordinary cohort is the crate's unit tests plus every +native integration test that is not harness infrastructure, a negative +control, a protocol vector suite or a real-server test; its generated cohort +is the conformance harness with `DIALCACHE_RUST_SUITE=generated` over the +complete corpus, witness evidence and Quint-generated vectors; its fixed +cohort is the harness with `DIALCACHE_RUST_SUITE=fixed` over the fixed +scenarios and checked-in vectors. Each mutant is compiled in release mode in +an isolated copy of the crate; dependencies build once and are shared through +`rust/target/semantic`. + Hosted runs shard each lane three ways with `MUTATION_SHARD=/`. A shard reruns the compile check, every unmodified baseline and the witness evaluation before its contiguous slice of the catalog, and writes an incomplete report under `shards/-of-/`. `make mutations-merge-ts` and -`make mutations-merge-go` (`merge-mutation-reports.mjs`) assemble the complete +`make mutations-merge-go` and `make mutations-merge-rust` +(`merge-mutation-reports.mjs`) assemble the complete report from those shards and refuse any inconsistency: a missing or duplicated shard, a shard that failed or claims completion, differing fingerprints or baseline results, or mutations that do not cover the catalog exactly once in @@ -167,7 +180,7 @@ The runner clears inherited trace selectors, uses the complete generated directo Each catalog entry's `requiredDetections` is a regression gate. The local mutation target and full workflow fail if a required fault survives. Newly detected faults remain visible as improvements; update the required set after inspecting the result. Source edits must match exactly once, so implementation drift requires reviewing the mutation rather than silently skipping it. -To expand assurance, add a test/doc-derived case and precise executable evidence, require a generated witness where appropriate, then add a representative fault for a previously unchallenged rule. Preserve gaps until execution closes them. Keep code coverage, source accounting, case evidence, and mutation detection as separate measurements. The current Go suite requires every shared profile, exported regression, fixed scenario and protocol case registered by the manifests, with an equivalent fault catalog in `go-mutations.json`. Broader interaction histories and larger domains remain separate assurance work. +To expand assurance, add a test/doc-derived case and precise executable evidence, require a generated witness where appropriate, then add a representative fault for a previously unchallenged rule. Preserve gaps until execution closes them. Keep code coverage, source accounting, case evidence, and mutation detection as separate measurements. The current Go and Rust suites require every shared profile, exported regression, fixed scenario and protocol case registered by the manifests, with equivalent fault catalogs in `go-mutations.json` and `rust-mutations.json`. Broader interaction histories and larger domains remain separate assurance work. ## Model properties and cross-language execution diff --git a/formal/VALIDATION.md b/formal/VALIDATION.md index 2e8b5945..b4796cd0 100644 --- a/formal/VALIDATION.md +++ b/formal/VALIDATION.md @@ -11,7 +11,7 @@ lists tool prerequisites and focused reproduction commands. | Routine implementation checks | `make check` | Native tests, coverage, package, docs and source audits | | Full portable acceptance | `make formal` | Rust model checks, generated histories, TS replay, shared witness evaluation, Go replay, exact completion inventories; no Java | | Finite symbolic rules | `make model-check` | Scheduled bounded checks with checksummed standalone Apalache; requires Java 21, `tar` and pinned Quint | -| Challenge implementation assertions | `make mutations` | Compiling semantic faults tested against both completed ports | +| Challenge implementation assertions | `make mutations` | Compiling semantic faults tested against every completed port | | Real server behavior | `make integration` | Redis, Valkey, Cluster and cross-language interoperability | | Explore another schedule sample | `make explore` | Separate source snapshot, recorded random seed, both-port replay | | Check composed profiles against their previous text | `make differential` | Lint baseline, then both-direction replay against the merge base with `origin/main`; fails on any disagreement or trace growth above the model's bound | @@ -118,11 +118,14 @@ not the catalog exactly once in order. Only the merged report is complete evidence; a shard report is never `complete`. Shard budgets are 30 minutes (TypeScript) and 40 (Go): the baselines plus four or five mutants, doubled for a slow runner. The Go mutation runner still bounds each `go test` invocation at -8 minutes to catch a hung mutant, not to pace a slow runner. Locally, +8 minutes to catch a hung mutant, not to pace a slow runner. The Rust lane +(`make mutations-rust`, 45-minute shard budget) compiles each mutant in +release mode and replays the generated cohort in about a minute; its runner +bounds each cargo invocation at 25 minutes. Locally, `MUTATION_SHARD=1/3 make mutations-ts` (then `2/3` and `3/3`) reproduces one shard under `.formal-traces/semantic/shards/1-of-3/`, and `make mutations-merge-ts` assembles the report that an unsharded -`make mutations-ts` writes; the Go targets mirror this. +`make mutations-ts` writes; the Go and Rust targets mirror this. The workflow's `formal-full` aggregate job retains a small `formal-summary` artifact for 90 days: both completion and context reports, the Go replay summary, the model diff --git a/formal/explore.mjs b/formal/explore.mjs index 34d975dd..48c9d3c1 100644 --- a/formal/explore.mjs +++ b/formal/explore.mjs @@ -167,7 +167,8 @@ export function nativeExplorationResult(language, text, context, directory, pack if (entry) failed.push(entry); else otherFailures.push(record.id); } const finish = records.at(-1); - if (finish?.kind !== 'finish' || (finish.status === 'failed') !== (failed.length + otherFailures.length > 0)) throw new Error('Rust report status disagrees with its case records.'); + if (finish?.kind !== 'finish') throw new Error('Rust report has no finish record: the harness crashed or timed out before completing.'); + if ((finish.status === 'failed') !== (failed.length + otherFailures.length > 0)) throw new Error('Rust report status disagrees with its case records.'); checkRustReplay(records.map(record => JSON.stringify(record.kind === 'case' ? { ...record, status: 'passed', message: undefined } : record.kind === 'finish' ? { ...record, status: 'passed', failed: 0 } : record)).join('\n'), inventory); startedAt = records[0]?.startedAt; finishedAt = finish.finishedAt; diff --git a/formal/go-parity.json b/formal/go-parity.json index cf65a783..70e795ce 100644 --- a/formal/go-parity.json +++ b/formal/go-parity.json @@ -10,7 +10,7 @@ "inputs": { "semanticCasesSha256": "7054107347f57a452e17de99f74310553b7af4350363c674e70ff8184d0610be", "executionSha256": "feee09a3487deb2694f8784b7e8d2d0ee28050dae03ac43ac2613608ce5ab1c6", - "sourceAuditSha256": "9a877f15ce14c557e52c3c0e11527cef4b56923b8ff472e595834f88d4bd9f29", + "sourceAuditSha256": "997c78fb4367e6da5b77d39866a178af5569e31b3ccfa97c732aa073caebf7ff", "quintCaseAuditSha256": "dcb576948a68a066e43259d4bee2b3cdadbe82e657ae9af99e4b21ed7b95dbb3", "featureCoverageSha256": "56eedfa865682db6ad8e73b8673c7dac1c30d301e9bce3390246d6994aba9dcb", "coverageWitnessesSha256": "8fc8ab123150fdec059f5b75f84172d39af80d08d7a4211fa88c63c5a3f2874b", @@ -18305,7 +18305,7 @@ ], "reviewedTestAndDocumentationAudit": { "path": "formal/source-audit.json", - "sha256": "9a877f15ce14c557e52c3c0e11527cef4b56923b8ff472e595834f88d4bd9f29", + "sha256": "997c78fb4367e6da5b77d39866a178af5569e31b3ccfa97c732aa073caebf7ff", "sourceFiles": 44, "status": "reviewed-applicability-not-execution", "limitations": "These rows review Go applicability for the public documentation and ordinary test files in source-audit.json sources, inheriting their C/W/E/B/X classifications. They map subjects to native artifacts without claiming one Go test per TypeScript assertion or turning documentation counts into executed coverage. The same audit hash also binds reviewedGuides for formal and Go documentation; those guide reviews are freshness/scope records, not additional behavioral applicability rows. Per-case Quint and replay evidence is recorded separately.", diff --git a/formal/measure-rust-semantics.mjs b/formal/measure-rust-semantics.mjs new file mode 100644 index 00000000..00711738 --- /dev/null +++ b/formal/measure-rust-semantics.mjs @@ -0,0 +1,256 @@ +import { createHash } from 'node:crypto'; +import { spawnSync } from 'node:child_process'; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { resolve, relative } from 'node:path'; +import { tmpdir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { fingerprintFiles, gateDetections, languages, partitionMutations, shardDirectory, shardFromArguments } from './mutation-reports.mjs'; + +// The Rust counterpart of measure-go-semantics.mjs: apply each catalogued +// fault to an isolated copy of the crate, require it to compile, and run three +// cohorts against it. Detection is an assertion failure; a compiler error, +// crash, timeout, missing report or incomplete run fails the measurement. +const root = fileURLToPath(new URL('../', import.meta.url)); +const hash = bytes => createHash('sha256').update(bytes).digest('hex'); +const json = file => JSON.parse(readFileSync(file, 'utf8')); + +// Test binaries that are the harness itself, its negative control, harness +// infrastructure, the protocol vector suites (the harness replays those +// vectors in the generated and fixed cohorts) or real servers are not +// ordinary tests. +export const infrastructureTestFile = /^(?:conformance|settlement_control|harness_infra|redis_integration|protocol_\w+)\.rs$/; + +// libtest prints one `test ... ` line per test and one +// `test result:` line per binary; `cargo test --no-fail-fast` runs every +// selected binary even after one fails. Names are qualified by their binary +// because unit and integration tests may share module paths. +export function evaluateCargoTestOutput(output, exitCode, expectedBinaries) { + const tests = new Map(); + let binary = '', results = 0, passed = 0, failed = 0; + for (const line of output.split('\n')) { + const running = /^\s*Running (?:unittests )?(\S+)/.exec(line); + if (running) { binary = running[1]; continue; } + const test = /^test (\S+) \.\.\. (ok|FAILED|ignored)/.exec(line); + if (test) { + const name = `${binary}::${test[1]}`; + if (tests.has(name)) throw new Error(`duplicate test execution ${name}`); + tests.set(name, test[2]); + continue; + } + const result = /^test result: (ok|FAILED)\. (\d+) passed; (\d+) failed; (\d+) ignored/.exec(line); + if (result) { results += 1; passed += Number(result[2]); failed += Number(result[3]); } + } + if (!results) throw new Error('no libtest result line: the ordinary cohort did not run'); + if (expectedBinaries !== undefined && results !== expectedBinaries) throw new Error(`expected ${expectedBinaries} test binaries to report, saw ${results}`); + const failingTests = [...tests].filter(([, outcome]) => outcome === 'FAILED').map(([name]) => name); + if (failed !== failingTests.length) throw new Error('libtest totals disagree with the listed outcomes'); + if (exitCode !== 0 && !failed) throw new Error(`cargo test exited ${exitCode} without a failing test: infrastructure failure`); + if (exitCode === 0 && failed) throw new Error('cargo test exited 0 with failing tests'); + return { state: failed ? 'detected' : 'survived', passed, failed, failingTests, + executedTests: [...tests].filter(([, outcome]) => outcome !== 'ignored').map(([name]) => name) }; +} + +// The harness writes one JSON object per line: a start header, one case per +// inventory id and a finish footer. A missing footer means the run crashed or +// timed out and is not evidence either way. +export function evaluateRustReport(text, exitCode) { + const records = text.trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); + if (!records.length) throw new Error('empty Rust harness report'); + const [start] = records; + if (start.kind !== 'start' || start.schemaVersion !== 1 || start.implementation !== 'rust') throw new Error('Rust harness report lacks its start header'); + const finish = records.at(-1); + if (finish.kind !== 'finish') throw new Error('Rust harness report has no finish record: the run crashed or timed out'); + const cases = records.slice(1, -1); + const seen = new Set(); + for (const record of cases) { + if (record.kind !== 'case' || typeof record.id !== 'string' || !['passed', 'failed'].includes(record.status)) throw new Error('malformed Rust harness case record'); + if (seen.has(record.id)) throw new Error(`duplicate case ${record.id}`); + seen.add(record.id); + } + if (!cases.length) throw new Error('Rust harness report has no cases'); + const failingTests = cases.filter(record => record.status === 'failed').map(record => record.id); + if (typeof finish.cases === 'number' && finish.cases !== cases.length) throw new Error('finish totals disagree with the case records'); + if (typeof finish.failed === 'number' && finish.failed !== failingTests.length) throw new Error('finish totals disagree with the failed cases'); + if (finish.status !== (failingTests.length ? 'failed' : 'passed')) throw new Error('finish status disagrees with the case records'); + if (exitCode === 0 && failingTests.length) throw new Error('harness exited 0 with failed cases'); + if (exitCode !== 0 && !failingTests.length) throw new Error(`harness exited ${exitCode} without a failed case: coverage or infrastructure failure`); + return { state: failingTests.length ? 'detected' : 'survived', passed: cases.length - failingTests.length, failed: failingTests.length, failingTests, + executedTests: cases.map(record => record.id) }; +} + +function union(generated, fixed) { + return { state: generated.failed + fixed.failed ? 'detected' : 'survived', passed: generated.passed + fixed.passed, + failed: generated.failed + fixed.failed, failingTests: [...generated.failingTests, ...fixed.failingTests], components: ['generated', 'fixed'] }; +} + +export function measureRustSemantics({ shard = { index: 1, count: 1 }, only } = {}) { + const language = languages.rust; + const reportRoot = resolve(root, language.output); + const output = shardDirectory(reportRoot, shard); + const started = Date.now(); + mkdirSync(reportRoot, { recursive: true }); + const report = { schemaVersion: 1, complete: false, ...(shard.count > 1 ? { shard: { index: shard.index, count: shard.count } } : {}), startedAt: new Date(started).toISOString(), baselines: {}, mutations: [] }; + const save = () => { report.elapsedSeconds = Math.round((Date.now() - started) / 1000); writeFileSync(resolve(output, 'report.json'), JSON.stringify(report, null, 2) + '\n'); }; + if (output !== reportRoot) { + writeFileSync(resolve(reportRoot, 'report.json'), JSON.stringify({ schemaVersion: 1, complete: false, startedAt: report.startedAt }, null, 2) + '\n'); + rmSync(output, { recursive: true, force: true }); + mkdirSync(output, { recursive: true }); + } + save(); rmSync(resolve(reportRoot, 'report.md'), { force: true }); + const workspace = mkdtempSync(resolve(tmpdir(), 'dialcache-rust-semantic-')); + const cargo = process.env.CARGO_BIN ?? 'cargo'; + // Bounds a hung mutant, not a slow runner: one release build of the crate + // and its test binaries plus the 7,000-case replay take a few minutes on a + // hosted runner. One timeout aborts the whole measurement. + const timeout = 1_500_000; + try { + // Copies keep repo-relative paths while mutations stay outside the shared + // checkout; the crate's build output is not an input and is not copied. + // go/ is copied because a crate unit test compares the invalidation + // script with go/redis_adapter.go byte for byte. + for (const path of ['formal', 'test', 'src', 'go']) cpSync(resolve(root, path), resolve(workspace, path), { recursive: true }); + const target = resolve(root, 'rust/target'); + cpSync(resolve(root, 'rust'), resolve(workspace, 'rust'), { recursive: true, filter: source => source !== target && !source.startsWith(`${target}/`) }); + const crate = resolve(workspace, 'rust'); + const catalogPath = resolve(workspace, language.catalog); + const catalog = json(catalogPath); + const typescript = json(resolve(workspace, 'formal/semantic-mutations.json')); + if (catalog.schemaVersion !== 1 || !Array.isArray(catalog.mutations) || catalog.mutations.length < 13) throw new Error('expected versioned Rust fault catalog with all 13 TypeScript counterparts'); + const originals = new Map(), ids = new Set(); + for (const mutation of catalog.mutations) { + if (!/^M\d+$/.test(mutation.id) || ids.has(mutation.id)) throw new Error('invalid/duplicate mutation ID'); + ids.add(mutation.id); + const counterpart = typescript.mutations.find(item => item.id === mutation.typescriptMutation); + if (!counterpart || counterpart.case !== mutation.case) throw new Error(`invalid TypeScript counterpart ${mutation.id}`); + if (!Array.isArray(mutation.edits) || !mutation.edits.length || !mutation.requiredDetections?.every(name => ['ordinary', 'generated', 'fixed', 'portable'].includes(name))) throw new Error(`invalid mutation ${mutation.id}`); + for (const edit of mutation.edits) { + if (!/^rust\/src\/[\w/-]+\.rs$/.test(edit.path) || !edit.before || edit.before === edit.after) throw new Error(`invalid production edit ${mutation.id}`); + const original = readFileSync(resolve(workspace, edit.path), 'utf8'); + if (original.split(edit.before).length !== 2) throw new Error(`${mutation.id}: anchor must occur exactly once; review source drift in ${edit.path}`); + originals.set(edit.path, original); + } + } + for (const counterpart of typescript.mutations) if (!catalog.mutations.some(m => m.typescriptMutation === counterpart.id)) throw new Error(`missing TypeScript counterpart ${counterpart.id}`); + const selected = only ? catalog.mutations.filter(m => only.includes(m.id)) : partitionMutations(catalog.mutations, shard); + if (only && selected.length !== only.length) throw new Error(`--only names unknown mutations: ${only.filter(id => !selected.some(m => m.id === id)).join(', ')}`); + if (shard.count > 1) report.shard = { index: shard.index, count: shard.count, mutationIds: selected.map(m => m.id) }; + // Ordinary tests: the library's unit tests and every integration binary + // that is not harness infrastructure, discovered from the crate so a new + // native test file joins the cohort automatically. + const ordinaryFiles = readdirSync(resolve(crate, 'tests')).filter(file => file.endsWith('.rs') && !infrastructureTestFile.test(file)).sort(); + if (!ordinaryFiles.length) throw new Error('no ordinary Rust integration tests found'); + const ordinaryTargets = ['--lib', ...ordinaryFiles.flatMap(file => ['--test', file.replace(/\.rs$/, '')])]; + const witnessDirectory = resolve(process.env.DIALCACHE_WITNESS_EVIDENCE_DIR ?? resolve(root, '.formal-traces/go-parity-witnesses')); + const corpus = { + DIALCACHE_MBT_TRACE_DIR: resolve(root, '.formal-traces/conformance'), + DIALCACHE_EFFECTS_TRACE_DIR: resolve(root, '.formal-traces/effects'), + DIALCACHE_FEATURE_TRACE_DIR: resolve(root, '.formal-traces/features'), + DIALCACHE_WITNESS_EVIDENCE_DIR: witnessDirectory, + }; + const cohorts = { + ordinary: { targets: ordinaryTargets }, + generated: { suite: 'generated', protocolCorpus: 'generated', env: corpus }, + fixed: { suite: 'fixed', protocolCorpus: 'fixed', env: {} }, + }; + const env = { ...process.env }; + for (const name of Object.keys(env)) if (name.startsWith('DIALCACHE_')) delete env[name]; + // Dependencies build once and are shared by every mutant and run through + // the checkout's cache directory; the crate copy itself lives at another + // path, so cargo rebuilds exactly the crate and its tests per mutant. + env.CARGO_TARGET_DIR = process.env.DIALCACHE_RUST_TARGET_DIR ?? resolve(root, 'rust/target/semantic'); + env.CARGO_TERM_COLOR = 'never'; + report.revision = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).stdout.trim(); + report.cargo = spawnSync(cargo, ['--version'], { cwd: crate, encoding: 'utf8' }).stdout?.trim(); + report.node = process.version; + report.catalogSha256 = hash(readFileSync(catalogPath)); + report.inputs = fingerprintFiles(workspace, language.inputs, { exclude: language.exclude }); + report.corpus = fingerprintFiles(root, ['.formal-traces/conformance', '.formal-traces/effects', '.formal-traces/features', '.formal-traces/regressions']); + report.witnesses = fingerprintFiles(witnessDirectory, ['.']); + report.sourceSha256 = Object.fromEntries([...originals].map(([path, text]) => [path, hash(text)])); + report.selections = { ordinary: ['lib', ...ordinaryFiles.map(file => file.replace(/\.rs$/, ''))], generated: 'conformance harness: DIALCACHE_RUST_SUITE=generated over the complete corpus, witness evidence and generated vectors', + fixed: 'conformance harness: DIALCACHE_RUST_SUITE=fixed over the fixed scenarios and checked-in vectors' }; + report.ordinaryFiles = ordinaryFiles; + const spawnCargo = (args, extraEnv) => spawnSync(cargo, args, { cwd: crate, env: { ...env, ...extraEnv }, encoding: 'utf8', timeout, maxBuffer: 256 * 1024 * 1024 }); + const compile = label => { + const result = spawnCargo(['test', '--release', '--all-features', '--no-run']); + writeFileSync(resolve(output, `${label}-compile.log`), (result.stdout ?? '') + (result.stderr ?? '')); + if (result.error || result.signal || result.status !== 0) throw new Error(`${label}: noncompiling mutant/baseline, not detection; see compile log`); + }; + const run = (label, cohort, baseline) => { + let parsed; + if (cohort === 'ordinary') { + const result = spawnCargo(['test', '--release', '--all-features', '--no-fail-fast', ...cohorts.ordinary.targets]); + writeFileSync(resolve(output, `${label}-${cohort}.log`), result.stdout ?? ''); + writeFileSync(resolve(output, `${label}-${cohort}.stderr.log`), result.stderr ?? ''); + if (result.error || result.signal) throw new Error(`${label}/${cohort}: runner infrastructure failed: ${result.error ?? result.signal}`); + parsed = evaluateCargoTestOutput(result.stdout ?? '', result.status, 1 + ordinaryFiles.length); + } else { + const reportPath = resolve(output, `${label}-${cohort}.jsonl`); + rmSync(reportPath, { force: true }); + const result = spawnCargo(['test', '--release', '--all-features', '--test', 'conformance'], + { ...cohorts[cohort].env, DIALCACHE_RUST_SUITE: cohorts[cohort].suite, DIALCACHE_PROTOCOL_CORPUS: cohorts[cohort].protocolCorpus, DIALCACHE_RUST_REPORT: reportPath }); + writeFileSync(resolve(output, `${label}-${cohort}.log`), result.stdout ?? ''); + writeFileSync(resolve(output, `${label}-${cohort}.stderr.log`), result.stderr ?? ''); + if (result.error || result.signal) throw new Error(`${label}/${cohort}: runner infrastructure failed: ${result.error ?? result.signal}`); + parsed = evaluateRustReport(existsSync(reportPath) ? readFileSync(reportPath, 'utf8') : '', result.status); + } + if (baseline && parsed.failed) throw new Error(`${cohort}: unmodified baseline must pass; see baseline log`); + if (!baseline && parsed.state === 'survived' && JSON.stringify([...parsed.executedTests].sort()) !== JSON.stringify([...report.baselines[cohort].executedTests].sort())) throw new Error(`${label}/${cohort}: incomplete surviving run`); + writeFileSync(resolve(output, `${label}-${cohort}.json`), JSON.stringify(parsed, null, 2) + '\n'); + return parsed; + }; + // Every shard compiles and measures every baseline itself: its evidence + // stands on the environment it ran in, and the merge refuses shards whose + // baselines differ. + compile('baseline'); + for (const cohort of Object.keys(cohorts)) { + report.baselines[cohort] = run('baseline', cohort, true); + console.log(`baseline ${cohort}: ${report.baselines[cohort].passed} passing tests`); save(); + } + report.baselines.portable = union(report.baselines.generated, report.baselines.fixed); + for (const mutation of selected) { + const editedPaths = new Set(); + try { + for (const edit of mutation.edits) { + const path = resolve(workspace, edit.path), current = readFileSync(path, 'utf8'); + if (current.split(edit.before).length !== 2) throw new Error(`${mutation.id}: overlapping edits`); + writeFileSync(path, current.replace(edit.before, edit.after)); editedPaths.add(edit.path); + } + compile(mutation.id); + const result = { id: mutation.id, case: mutation.case, description: mutation.description, cohorts: {} }; + for (const cohort of Object.keys(cohorts)) result.cohorts[cohort] = run(mutation.id, cohort, false); + result.cohorts.portable = union(result.cohorts.generated, result.cohorts.fixed); + report.mutations.push(result); + console.log(`${mutation.id}: ${Object.entries(result.cohorts).map(([name, value]) => `${name}=${value.state}(${value.failed})`).join(', ')}`); save(); + } finally { for (const path of editedPaths) writeFileSync(resolve(workspace, path), originals.get(path)); } + } + if (only) { + // A filtered run is a local diagnostic, never evidence: it stays incomplete + // and is not gated. + report.partial = only; + save(); + console.log(`Measured ${selected.length} selected mutations without gating: ${relative(root, output)}/report.json`); + return report; + } + if (shard.count > 1) { + gateDetections(language, report, selected, { directory: root, summarize: false }); + save(); + console.log(`Shard ${shard.index}/${shard.count} measured ${selected.length} mutations: ${relative(root, output)}/report.json; merge with node formal/merge-mutation-reports.mjs rust`); + return report; + } + gateDetections(language, report, catalog.mutations, { directory: root }); + save(); + writeFileSync(resolve(output, 'report.md'), language.markdown(report)); + return report; + } catch (error) { report.error = String(error); save(); throw error; } + finally { rmSync(workspace, { recursive: true, force: true }); } +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + try { + const only = process.argv.slice(2).filter(argument => argument.startsWith('--only=')).flatMap(argument => argument.slice('--only='.length).split(',').filter(Boolean)); + const shard = shardFromArguments(process.argv.slice(2).filter(argument => !argument.startsWith('--only='))); + if (only.length && shard.count > 1) throw new Error('--only cannot be combined with --shard'); + measureRustSemantics({ shard, only: only.length ? only : undefined }); + } catch (error) { console.error(error); process.exitCode = 1; } +} diff --git a/formal/merge-mutation-reports.mjs b/formal/merge-mutation-reports.mjs index 1085028b..12a2fd3c 100644 --- a/formal/merge-mutation-reports.mjs +++ b/formal/merge-mutation-reports.mjs @@ -111,7 +111,7 @@ export function readShardReports(directory) { // Any refusal leaves an incomplete report with the reason in its place. export function mergeMutationReports(name, { directory = root, shardsDirectory, outputDirectory } = {}) { const language = languages[name]; - if (!language) throw new Error(`Expected language ts or go; got ${name}`); + if (!language) throw new Error(`Expected language ts, go or rust; got ${name}`); const output = resolve(directory, outputDirectory ?? language.output); const shards = resolve(directory, shardsDirectory ?? resolve(output, 'shards')); const startedAt = new Date().toISOString(); @@ -123,7 +123,7 @@ export function mergeMutationReports(name, { directory = root, shardsDirectory, try { const catalogText = readFileSync(resolve(directory, language.catalog)); const catalog = JSON.parse(catalogText); - merged = mergeShardReports(language, readShardReports(shards), { catalog, catalogSha256: sha256(catalogText), inputs: fingerprintFiles(directory, language.inputs) }); + merged = mergeShardReports(language, readShardReports(shards), { catalog, catalogSha256: sha256(catalogText), inputs: fingerprintFiles(directory, language.inputs, { exclude: language.exclude }) }); gateDetections(language, merged, catalog.mutations, { directory }); } catch (error) { const failed = merged ?? { schemaVersion: 1, complete: false, startedAt }; @@ -140,7 +140,7 @@ export function mergeMutationReports(name, { directory = root, shardsDirectory, if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { const [name, shardsDirectory, ...extra] = process.argv.slice(2); if (!languages[name] || extra.length) { - console.error('Usage: node formal/merge-mutation-reports.mjs [shard-directory]'); + console.error('Usage: node formal/merge-mutation-reports.mjs [shard-directory]'); process.exitCode = 2; } else { try { diff --git a/formal/mutation-reports.mjs b/formal/mutation-reports.mjs index 2c15ee9d..be32b1e6 100644 --- a/formal/mutation-reports.mjs +++ b/formal/mutation-reports.mjs @@ -1,5 +1,5 @@ import { createHash } from 'node:crypto'; -import { readFileSync, readdirSync } from 'node:fs'; +import { readFileSync, readdirSync, statSync } from 'node:fs'; import { resolve } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -45,12 +45,17 @@ export function shardDirectory(output, shard) { // Hash the reviewed inputs, including uncommitted edits and exact file bytes. // Git revision alone cannot identify an exploratory run from a dirty worktree. -export function fingerprintFiles(directory, paths) { +// `exclude` names build output beneath an input (the Rust target directory) +// that is neither reviewed nor stable. +export function fingerprintFiles(directory, paths, { exclude = [] } = {}) { const files = []; + const skipped = new Set(exclude); const visit = path => { + if (skipped.has(path)) return; + if (statSync(resolve(directory, path)).isFile()) { files.push(path); return; } for (const entry of readdirSync(resolve(directory, path), { withFileTypes: true })) { if (entry.isDirectory()) visit(`${path}/${entry.name}`); - else if (entry.isFile()) files.push(`${path}/${entry.name}`); + else if (entry.isFile() && !skipped.has(`${path}/${entry.name}`)) files.push(`${path}/${entry.name}`); } }; for (const path of paths) visit(path); @@ -83,6 +88,7 @@ export function typescriptDetection(mutations, cases) { return { all: score(mutations), behavioral: score(mutations.filter(m => !protocol(m))), protocol: score(mutations.filter(protocol)) }; } +// The four-cohort score shared by the Go and Rust measurements. export function goDetection(mutations) { return Object.fromEntries(['ordinary', 'generated', 'fixed', 'portable'].map(cohort => [cohort, { detected: mutations.filter(m => m.cohorts[cohort].state === 'detected').length, total: mutations.length, @@ -111,6 +117,14 @@ function goMarkdown(report) { 'Full JSON records snapshot/corpus/witness fingerprints, selected tests, actual passing/failing leaf counts, and assertion diagnostics. Compilation errors, crashes, timeouts, missing witnesses, and skipped executions cannot count as detections.', ''].join('\n'); } +function rustMarkdown(report) { + return ['# Rust semantic mutation measurement', '', `Completed in ${report.elapsedSeconds}s. Counts measure this named fault catalog and exact corpus, not universal equivalence.`, '', + ...shardsMarkdown(report), + '| Mutation | Contract case | Ordinary | Quint generated | Fixed supplement | Full portable |', '| --- | --- | --- | --- | --- | --- |', + ...report.mutations.map(m => `| ${m.id} | ${m.case} | ${m.cohorts.ordinary.state} | ${m.cohorts.generated.state} | ${m.cohorts.fixed.state} | ${m.cohorts.portable.state} |`), '', + 'Ordinary is the crate\'s unit and native tests; generated is the conformance harness over the complete corpus and witness evidence; fixed is the harness over the fixed scenarios and checked-in vectors. Full JSON records snapshot/corpus/witness fingerprints, cohort selections, actual passing/failing counts and assertion diagnostics. Compilation errors, crashes, timeouts, missing reports and incomplete runs cannot count as detections.', ''].join('\n'); +} + // Only a merged report carries `shards`; the single run's markdown is unchanged. function shardsMarkdown(report) { if (!report.shards) return []; @@ -127,6 +141,10 @@ export const languages = { markdown: typescriptMarkdown, recordsRegressions: false }, go: { name: 'Go', output: '.formal-traces/go-semantic', catalog: 'formal/go-mutations.json', inputs: ['formal', 'go', 'test', 'src'], detection: mutations => goDetection(mutations), markdown: goMarkdown, recordsRegressions: true }, + // The crate's unit tests read go/redis_adapter.go (script byte equality), so + // the Go source is an input of the ordinary cohort as well. + rust: { name: 'Rust', output: '.formal-traces/rust-semantic', catalog: 'formal/rust-mutations.json', inputs: ['formal', 'rust', 'test', 'src', 'go'], exclude: ['rust/target'], + detection: mutations => goDetection(mutations), markdown: rustMarkdown, recordsRegressions: true }, }; // The same gate for a shard (its own catalog entries, no summary), the diff --git a/formal/rust-mutations.json b/formal/rust-mutations.json new file mode 100644 index 00000000..b425b757 --- /dev/null +++ b/formal/rust-mutations.json @@ -0,0 +1,302 @@ +{ + "schemaVersion": 1, + "mutations": [ + { + "id": "M01", + "case": "C45.maximum-age-exclusive", + "description": "Accept a retained candidate at the exact maximum age after decoding", + "typescriptMutation": "M01", + "edits": [ + { + "path": "rust/src/execution.rs", + "before": " if !valid || age as u64 >= max_age {\n self.recovery_event(RecoveryOutcome::Miss, None);\n return None;\n }\n self.recovery_event(RecoveryOutcome::Served, Some(age));", + "after": " if !valid || age as u64 > max_age {\n self.recovery_event(RecoveryOutcome::Miss, None);\n return None;\n }\n self.recovery_event(RecoveryOutcome::Served, Some(age));" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M02", + "case": "C45.recheck-after-decode", + "description": "Reuse the pre-decode age instead of observing age after asynchronous decoding", + "typescriptMutation": "M02", + "edits": [ + { + "path": "rust/src/execution.rs", + "before": " };\n let (age, valid) = self.frame_age(frame, Layer::Remote);", + "after": " };\n // Fault injection: retain the pre-decode age and validity." + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M03", + "case": "C25.late-source-rejected", + "description": "Accept an externally settled result after its elapsed deadline when the timer has not run", + "typescriptMutation": "M03", + "edits": [ + { + "path": "rust/src/deadline.rs", + "before": " Either::Left((value, _timer)) => {\n if since(clock, started) < budget {", + "after": " Either::Left((value, _timer)) => {\n if true {" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M04", + "case": "C34.second-fence", + "description": "Skip the tracked fence recheck after serialization", + "typescriptMutation": "M04", + "edits": [ + { + "path": "rust/src/execution.rs", + "before": " if let Some(fence) = fence {\n if stamp <= fence {", + "after": " if let Some(fence) = fence {\n if false && stamp <= fence {" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M05", + "case": "C03.detached-bypass", + "description": "Treat a detached context whose outer scope closed as still enabled", + "typescriptMutation": "M05", + "edits": [ + { + "path": "rust/src/scope.rs", + "before": " self.enabled && self.owner.as_ref().is_some_and(|owner| owner.is_live())", + "after": " self.enabled" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M06", + "case": "W04.strict-fence", + "description": "Accept a tracked frame timestamp equal to its watermark", + "typescriptMutation": "M06", + "edits": [ + { + "path": "rust/src/protocol/frame.rs", + "before": " if stamp <= fence {\n return miss(MissReason::WatermarkFenced);", + "after": " if stamp < fence {\n return miss(MissReason::WatermarkFenced);" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M07", + "case": "C51.changed-confirmation", + "description": "Confirm an unequal shadow result even when C1 contains different payload bytes", + "typescriptMutation": "M07", + "edits": [ + { + "path": "rust/src/shadow.rs", + "before": " ReadResult::Hit(confirmed) if confirmed.payload.bytes == frame.payload.bytes => {}", + "after": " ReadResult::Hit(_) => {}" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M08", + "case": "C60.invalid-logging-policy", + "description": "Enable mismatch logging after a malformed runtime logging flag", + "typescriptMutation": "M08", + "edits": [ + { + "path": "rust/src/policy.rs", + "before": " shadow.log_mismatches = false;\n shadow.logging_config_error = true;", + "after": " shadow.log_mismatches = true;\n shadow.logging_config_error = true;" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M09", + "case": "C60.logging-default-off", + "description": "Enable mismatch logging when its flag is omitted", + "typescriptMutation": "M09", + "edits": [ + { + "path": "rust/src/policy.rs", + "before": " let mut shadow = ResolvedShadow::default();", + "after": " let mut shadow = ResolvedShadow {\n log_mismatches: true,\n ..ResolvedShadow::default()\n };" + } + ], + "requiredDetections": [ + "generated", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M10", + "case": "C31.tracked-no-local-fill", + "description": "Publish an unvalidated tracked source value directly into local storage", + "typescriptMutation": "M10", + "edits": [ + { + "path": "rust/src/execution.rs", + "before": " if local_miss && !self.identity.tracked {", + "after": " if local_miss {" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M11", + "case": "C09.fixed-local-ttl", + "description": "Renew the local insertion TTL on every cache hit", + "typescriptMutation": "M11", + "edits": [ + { + "path": "rust/src/local.rs", + "before": " Ok(match self.entries.get(key) {\n Some(entry) => LocalRead::Live(entry.value.clone()),\n None => LocalRead::Absent,\n })", + "after": " Ok(match self.entries.get_mut(key) {\n Some(entry) => {\n entry.inserted_ms = now_ms;\n LocalRead::Live(entry.value.clone())\n }\n None => LocalRead::Absent,\n })" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + }, + { + "id": "M12", + "case": "W02.sorted-arguments", + "description": "Reverse the prescribed argument-name ordering", + "typescriptMutation": "M12", + "edits": [ + { + "path": "rust/src/identity.rs", + "before": " pairs.sort_by(|left, right| compare_utf16(&left.0, &right.0));", + "after": " pairs.sort_by(|left, right| compare_utf16(&right.0, &left.0));" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "generated", + "portable" + ] + }, + { + "id": "M13", + "case": "C54.expired-before-start", + "description": "Dispatch dark shadow reads after the job deadline already elapsed before deferred work starts", + "typescriptMutation": "M13", + "edits": [ + { + "path": "rust/src/shadow.rs", + "before": " if expired() {\n return Verdict::of(ShadowOutcome::Timeout);\n }\n let mut fill = false;", + "after": " if false && expired() {\n return Verdict::of(ShadowOutcome::Timeout);\n }\n let mut fill = false;" + } + ], + "requiredDetections": [ + "generated", + "fixed", + "portable" + ], + "typescriptRequiredDetections": [ + "ordinary", + "generated", + "portable" + ] + } + ] +} diff --git a/formal/source-audit.json b/formal/source-audit.json index 9e797660..ceea71ca 100644 --- a/formal/source-audit.json +++ b/formal/source-audit.json @@ -1287,7 +1287,7 @@ }, { "path": "formal/PORTING.md", - "sha256": "204d3c424f89c3014641e1027d464bacc7c6099ef3ee47ea373b30db92062841", + "sha256": "763f323d0d09fc3d584f5791f9bea232591c20aea16296d78fef3996d7510a82", "entries": [ {"line":1,"title":"Implementing another DialCache port"}, {"line":14,"title":"What a port supplies"}, @@ -1346,7 +1346,7 @@ }, { "path": "formal/README.md", - "sha256": "d612ebd99ef15cfe0540524163e5fb904bb416171909aa20b5df24894d3f242c", + "sha256": "fa1222a3b0ceb6516d9e111e58f334f14a5d15ea3e83d70720b9bf9f6906a821", "entries": [ {"line":1,"title":"Executable DialCache specification"}, {"line":7,"title":"Start with your task"}, @@ -1362,7 +1362,7 @@ }, { "path": "formal/SEMANTIC-COVERAGE.md", - "sha256": "36dfebb6bfe76998f42ae0880ba5f9a984e6c5cd4376c2aef55b189cf12cdc65", + "sha256": "89cf581c6701c05ffd946257def034716ddba9e42c6177cf63dea69234ea2059", "entries": [ {"line":1,"title":"Measuring semantic coverage"}, {"line":5,"title":"Evidence inventory"}, @@ -1371,7 +1371,7 @@ {"line":110,"title":"Behavioral mutation comparison"}, {"line":124,"title":"Recorded measurements"}, {"line":130,"title":"Reproduction and CI"}, - {"line":172,"title":"Model properties and cross-language execution"} + {"line":185,"title":"Model properties and cross-language execution"} ], "review": { "kind": "coverage-guide", @@ -1501,15 +1501,15 @@ }, { "path": "formal/VALIDATION.md", - "sha256": "79d61df9534e7a1db8a1b4819076e7cc7a4e7aff43281756e9135d95f86e1308", + "sha256": "42143a8e6f9882ae782e894e0aca0712d73fc03502ff2cf9ce95708e060df395", "entries": [ {"line":1,"title":"Validation and evidence"}, {"line":7,"title":"Choosing a run"}, {"line":67,"title":"Reading a completion report"}, {"line":85,"title":"Mutation evidence"}, - {"line":146,"title":"Exploratory runs"}, - {"line":194,"title":"A retained sampling regression"}, - {"line":231,"title":"Historical results"} + {"line":149,"title":"Exploratory runs"}, + {"line":197,"title":"A retained sampling regression"}, + {"line":234,"title":"Historical results"} ], "review": { "kind": "tooling-guide", diff --git a/formal/validation.mjs b/formal/validation.mjs index f885b8ce..bd195336 100644 --- a/formal/validation.mjs +++ b/formal/validation.mjs @@ -10,7 +10,7 @@ const replayTests = ['test/formal-conformance.test.ts', 'test/formal-effects.tes const aggregateTargets = { check: ['check-ts', 'check-go', 'check-rust', 'docs', 'audit'], formal: ['formal-check', 'formal-generate', 'formal-ts', 'formal-go', 'formal-rust'], - mutations: ['mutations-ts', 'mutations-go'], + mutations: ['mutations-ts', 'mutations-go', 'mutations-rust'], integration: ['integration-ts', 'integration-go', 'integration-rust'], ci: ['check', 'package-floor', 'formal', 'model-check', 'integration', 'mutations'], }; @@ -31,13 +31,15 @@ export const targetDescriptions = { 'fixtures-check': 'Recompute every committed model-derived artifact with pinned Quint', 'kernel-fixtures': 'Typecheck the kernel library fixtures (test/fixtures/kernel) and run every run they declare', differential: 'Check the composition lint baseline, then replay every composed profile against its reference corpus (merge base with DIFFERENTIAL_REFERENCE, default origin/main) in both directions', - explore: 'Explore a new recorded seed and replay both ports in an isolated source snapshot', + explore: 'Explore a new recorded seed and replay every port in an isolated source snapshot', 'model-check': 'Symbolically verify the scheduled finite rules with pinned Quint/Apalache (Java 21)', - mutations: 'Measure TypeScript and Go semantic mutations over the generated corpus and shared witness evidence', + mutations: 'Measure TypeScript, Go and Rust semantic mutations over the generated corpus and shared witness evidence', 'mutations-ts': 'Measure TypeScript semantic mutations over the generated corpus (MUTATION_SHARD=/ measures one shard)', 'mutations-go': 'Measure Go semantic mutations over the generated corpus and shared witness evidence (MUTATION_SHARD=/ measures one shard)', + 'mutations-rust': 'Measure Rust semantic mutations over the generated corpus and shared witness evidence (MUTATION_SHARD=/ measures one shard)', 'mutations-merge-ts': 'Merge TypeScript mutation shards into the complete report; refuses inconsistent or missing shards', 'mutations-merge-go': 'Merge Go mutation shards into the complete report; refuses inconsistent or missing shards', + 'mutations-merge-rust': 'Merge Rust mutation shards into the complete report; refuses inconsistent or missing shards', integration: 'Run real TypeScript, Go and Rust Redis/Valkey/Cluster integration checks', 'integration-ts': 'Run TypeScript real integration checks', 'integration-go': 'Run Go real integration and interoperability checks with race detection', @@ -55,12 +57,12 @@ export function expandTargets(target) { // its catalog; the merge target later assembles the complete report. Only the // two leaf lanes accept it: an aggregate that silently ignored it would run the // complete measurement the caller did not ask for. -const shardedTargets = ['mutations-ts', 'mutations-go']; +const shardedTargets = ['mutations-ts', 'mutations-go', 'mutations-rust']; export function mutationShardArguments(target, environment = process.env) { const value = environment.MUTATION_SHARD; if (value === undefined) return []; if (!shardedTargets.includes(target)) { - if (expandTargets(target).some(name => shardedTargets.includes(name))) throw new Error(`MUTATION_SHARD=${value} applies only to make mutations-ts and make mutations-go; unset it to run the complete measurement with make ${target}.`); + if (expandTargets(target).some(name => shardedTargets.includes(name))) throw new Error(`MUTATION_SHARD=${value} applies only to make mutations-ts, make mutations-go and make mutations-rust; unset it to run the complete measurement with make ${target}.`); return []; } // The measurement scripts parse the same value; one implementation decides @@ -96,7 +98,6 @@ export function validationPlan(target, { directory = root, environment = process const node = (label, ...args) => ({ label, command: runnerNode, args }); const pnpm = (label, ...args) => ({ label, command: 'corepack', args: ['pnpm', ...args] }); const go = (label, ...args) => ({ label, command: 'go', args: ['-C', 'go', ...args] }); - // Every step spawns from the repository root; the manifest path selects the crate. // Cargo runs inside rust/ so rustup resolves rust/rust-toolchain.toml; the // crate's tests locate the repository through CARGO_MANIFEST_DIR, not cwd. const cargo = (label, subcommand, ...args) => ({ label, command: 'cargo', args: [subcommand, ...args], cwd: 'rust' }); @@ -181,15 +182,17 @@ export function validationPlan(target, { directory = root, environment = process { ...node('Adapt Rust native assertion report', 'formal/conformance-adapters.mjs', 'rust', '.formal-traces/rust-replay.jsonl', reportPath('rust', 'context')), stdoutFile: reportPath('rust', 'completion') }, completion('rust')], 'mutations-ts': [node('Measure TypeScript semantic mutations', 'formal/measure-semantics.mjs', ...shard)], 'mutations-go': [node('Measure Go semantic mutations', 'formal/measure-go-semantics.mjs', ...shard)], + 'mutations-rust': [node('Measure Rust semantic mutations', 'formal/measure-rust-semantics.mjs', ...shard)], // The merge needs neither Quint nor Go: it reads shard reports, checks // them against each other and this checkout, and writes the complete report. 'mutations-merge-ts': [node('Merge TypeScript mutation shards', 'formal/merge-mutation-reports.mjs', 'ts')], 'mutations-merge-go': [node('Merge Go mutation shards', 'formal/merge-mutation-reports.mjs', 'go')], + 'mutations-merge-rust': [node('Merge Rust mutation shards', 'formal/merge-mutation-reports.mjs', 'rust')], 'integration-ts': [pnpm('Run TypeScript Redis/Valkey/Cluster integrations', 'test:integration')], 'integration-go': [{ ...go('Run Go Redis/Valkey/Cluster and TypeScript interoperability', 'test', '-race', '-tags', 'integration', '-count=1', '-run', '^TestRedisIntegration$', '-json', './...'), stdoutFile: '.formal-traces/go-integration.jsonl' }], - // The Rust integration binary skips itself unless this variable is set, so - // a plain cargo test never needs Docker. - 'integration-rust': [{ ...cargo('Run Rust Redis/Valkey/Cluster integrations', 'test', '--all-features', '--test', 'redis_integration'), env: { DIALCACHE_RUST_INTEGRATION: '1' } }], + // The Rust integration tests are #[ignore]d, so a plain cargo test reports + // them as ignored and never needs Docker; this lane runs exactly them. + 'integration-rust': [cargo('Run Rust Redis/Valkey/Cluster integrations', 'test', '--all-features', '--test', 'redis_integration', '--', '--ignored')], 'package-floor': [{ label: 'Require a built package for floor checks', requireFile: 'dist/index.js', failureHint: 'Build first with make check-ts, or run make ci with NODE22_BIN set.' }, { label: 'Check Node 22.15 zstd round trip and output ceiling', command: node22, args: ['--eval', floorSmoke], env: { PATH: floorEnvironment(environment, node22).PATH } }, { label: 'Check packed package on Node 22.15', command: node22, args: ['scripts/test-package.mjs'], env: { PATH: floorEnvironment(environment, node22).PATH } }], @@ -214,7 +217,7 @@ export function checkPrerequisites(target, { directory = root, environment = pro const version = probe('go', ['version'], { directory, environment }); if (!/^go version go1\.27\.1\s/.test(version)) throw new Error(`Validation requires Go 1.27.1; found ${version}. Put the pinned Go toolchain on PATH.`); } - if (targets.some(name => ['check-rust', 'smoke', 'formal-rust', 'integration-rust'].includes(name))) { + if (targets.some(name => ['check-rust', 'smoke', 'formal-rust', 'integration-rust', 'mutations-rust', 'explore'].includes(name))) { const version = probe('cargo', ['--version'], { directory: resolve(directory, 'rust'), environment }); if (!/^cargo 1\.98\.1(?:\s|$)/.test(version)) throw new Error(`Validation requires cargo 1.98.1; found ${version}. Install Rust 1.98.1 (rustup reads rust/rust-toolchain.toml) and put it on PATH.`); } diff --git a/rust/README.md b/rust/README.md index fd1605ec..b32b0de5 100644 --- a/rust/README.md +++ b/rust/README.md @@ -150,8 +150,9 @@ or distribution and a namespace. for `ConnectionManager`, `MultiplexedConnection` and cluster connections, routing tracked reads to slot primaries and sharing the invalidation script and frame codec with the other ports. The `redis_integration` test replays -every invalidation vector against real Redis, Valkey and Cluster servers when -`DIALCACHE_RUST_INTEGRATION=1` is set and Docker is available. +every invalidation vector against real Redis, Valkey and Cluster servers; its +tests are `#[ignore]`d, and `make integration-rust` runs them where Docker is +available. ## Validation and reproducing a trace @@ -165,8 +166,15 @@ make check-rust # fmt, clippy, unit tests, protocol vectors, fixed scenar make integration-rust # Real Redis, Valkey and Cluster servers through Docker, plus every invalidation vector make formal # Quint model checks, full corpus, then TypeScript, Go and Rust replay make formal-rust # Complete prepared Rust replay of the generated corpus +make mutations-rust # Measure the Rust fault catalog (formal/rust-mutations.json) against the replay ``` +`make mutations-rust` applies each catalogued single-site fault to an isolated +copy of the crate and requires the conformance harness to detect it +(`DIALCACHE_RUST_SUITE=generated` for the Quint-generated evidence, +`DIALCACHE_RUST_SUITE=fixed` for the fixed scenarios); see +[SEMANTIC-COVERAGE.md](../formal/SEMANTIC-COVERAGE.md). + Without overrides, `cargo test --all-features --test conformance` replays the committed smoke histories, every fixed scenario and every protocol vector. The same `DIALCACHE_*_TRACE_DIR` / `_TRACE_FILE` selectors as Go replay a diff --git a/rust/src/engine.rs b/rust/src/engine.rs index 338b3616..0e6d56e2 100644 --- a/rust/src/engine.rs +++ b/rust/src/engine.rs @@ -327,7 +327,9 @@ impl DialCacheBuilder { /// Validate the configuration and create the instance. Rejects a /// namespace containing `{` or `}`, a read budget outside /// `1..=2_147_483_647` ms, a zero shadow cap, an invalid compression - /// config, or a missing runtime when the `tokio` feature is off. + /// config, or no usable runtime: with the `tokio` feature, no tokio + /// runtime is current and none was supplied through `runtime` or + /// `runtime_arc`; without it, no runtime was supplied. pub fn build(self) -> Result { if self.namespace.contains(['{', '}']) { return Err(ConfigError::invalid( diff --git a/rust/src/execution.rs b/rust/src/execution.rs index 58cfbcbf..0f526cd9 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -429,10 +429,16 @@ impl Execution { }); let result = self.clone().shared(Layer::RequestLocal).await; if let Ok(value) = &result { - let mut state = owner.state.lock(); - if state.live { - state.memo.insert(self.keys.logical.clone(), value.clone()); - } + let displaced = { + let mut state = owner.state.lock(); + if state.live { + state.memo.insert(self.keys.logical.clone(), value.clone()) + } else { + None + } + }; + // A replaced memo value drops here, outside the lock. + drop(displaced); } result } diff --git a/rust/src/observe.rs b/rust/src/observe.rs index 3a8ae64a..b958e54c 100644 --- a/rust/src/observe.rs +++ b/rust/src/observe.rs @@ -236,7 +236,10 @@ pub struct OutcomeLabels { /// sizes report bytes. #[derive(Debug, Clone, PartialEq)] pub enum Event { - /// One lookup of an enabled cache layer, emitted before its result is known. + /// One lookup of an enabled cache layer, hit or miss. Emitted before the + /// result for the remote layer and after it for the request-local and + /// process-local layers; a layer whose read failed emits `Error` and + /// `Disabled` instead. Request { /// The layer that was looked up. labels: Labels, @@ -344,10 +347,13 @@ pub enum Event { seconds: f64, }, /// Time until the source settled or its deadline elapsed, once per - /// source invocation. + /// source invocation that served the caller (shadow re-validation runs + /// the source without it). Fallback { - /// The deepest layer traversed before the source ran, or - /// [`Layer::Noop`] when caching was bypassed. + /// [`Layer::RequestLocal`] when the request memo was consulted, + /// [`Layer::Remote`] when the remote layer was consulted, otherwise + /// [`Layer::Local`] (even with the local layer disabled), and + /// [`Layer::Noop`] only when the call bypassed caching. labels: Labels, /// Elapsed time from invoking the source until it settled or its /// deadline fired. @@ -437,8 +443,10 @@ pub struct ShadowMismatchDetails { pub enum LogLevel { /// Verbose diagnostics; no [`LogEvent`] maps here today. Debug, - /// A fail-open diagnostic: the call still produced a result from another - /// layer or the source. + /// A diagnostic on a path where the cache itself added no failure: a + /// layer failed open and the call went on to another layer or the + /// source, or an explicit invalidation or a recovery decode failed and + /// the operation reports that failure. Warn, /// A whole path is broken: key construction or the process-local store. Error, diff --git a/rust/src/policy.rs b/rust/src/policy.rs index 44dab1fb..495e23e6 100644 --- a/rust/src/policy.rs +++ b/rust/src/policy.rs @@ -390,7 +390,9 @@ pub struct ResolvedShadow { /// was omitted or malformed. pub log_mismatches: bool, /// The shadow ramp leaf was present but invalid: shadow work is off and a - /// `config_resolution` error is emitted on the remote layer. + /// `config_resolution` error is emitted on the remote layer whenever a + /// shadow job would otherwise have been scheduled (a remote hit or a + /// ramped-down remote). pub config_error: bool, /// Recorded only if a job is admitted. pub logging_config_error: bool, diff --git a/rust/src/prometheus.rs b/rust/src/prometheus.rs index a9d3b883..f103a12d 100644 --- a/rust/src/prometheus.rs +++ b/rust/src/prometheus.rs @@ -11,8 +11,11 @@ //! series. The `prometheus` crate reports a duplicate registration without a //! handle to the existing collector, so a second `new` for the same registry //! and prefix is a [`PrometheusError::Conflict`], as is an externally -//! registered collector under one of the DialCache names; the constructor -//! then leaves the registry as it found it. +//! registered collector under one of the DialCache names. A failed +//! constructor unregisters every collector it had registered, so no series is +//! left behind; the `prometheus` crate nevertheless remembers each such name +//! with the DialCache schema for the registry's lifetime, so only a collector +//! with the same schema can take that name later. use std::fmt; use std::sync::Arc; @@ -206,6 +209,8 @@ pub enum PrometheusError { /// A collector with this name is already registered: by another observer /// (clone that observer instead of constructing a second one), by someone /// else, or with another schema. Use a unique prefix or another registry. + /// The collectors registered before the collision were unregistered + /// again; their names stay bound to the DialCache schema in that registry. Conflict { /// The fully qualified collector name that collided. name: String, diff --git a/rust/src/protocol/envelope.rs b/rust/src/protocol/envelope.rs index cb9ece99..c56f192b 100644 --- a/rust/src/protocol/envelope.rs +++ b/rust/src/protocol/envelope.rs @@ -77,10 +77,12 @@ pub struct CompressionWriteResult { /// What [`decompress_payload`] produced from a stored payload. #[derive(Debug, Clone, PartialEq, Eq)] pub struct CompressionReadResult { - /// The decoded payload, or the input unchanged when it passed through or - /// zstd rejected it. + /// The decoded payload; the input with its `0x00` escape prefix removed + /// when it was escaped raw output; or the input unchanged when it passed + /// through untouched or zstd rejected it. pub payload: Payload, - /// `None` when the payload passed through untouched. + /// `None` when no zstd envelope was involved: the payload passed through + /// untouched or only had its escape prefix removed. pub outcome: Option, } diff --git a/rust/src/redis.rs b/rust/src/redis.rs index b141497b..2cc724a2 100644 --- a/rust/src/redis.rs +++ b/rust/src/redis.rs @@ -441,11 +441,14 @@ mod tests { args: Vec>, } - /// Replays scripted replies in order; a missing reply means "hang". + /// Replays scripted replies in order. A dispatch beyond the script fails + /// with a client error so a test that expected no dispatch fails loudly; + /// [`Scripted::hanging`] makes it wait forever instead. #[derive(Default)] struct Scripted { calls: Mutex>, replies: Mutex>>, + hang_when_exhausted: bool, } impl Scripted { @@ -453,6 +456,16 @@ mod tests { Arc::new(Scripted { calls: Mutex::new(Vec::new()), replies: Mutex::new(replies.into_iter().collect()), + hang_when_exhausted: false, + }) + } + + /// A connection whose replies never arrive, for cancellation tests. + fn hanging() -> Arc { + Arc::new(Scripted { + calls: Mutex::new(Vec::new()), + replies: Mutex::new(VecDeque::new()), + hang_when_exhausted: true, }) } @@ -471,7 +484,14 @@ mod tests { self.calls.lock().push(Call { primary, args }); match self.replies.lock().pop_front() { Some(reply) => future::ready(reply).boxed(), - None => future::pending().boxed(), + // A dispatch beyond the script is a test failure, not a hang, + // unless the test asked for a reply that never arrives. + None if self.hang_when_exhausted => future::pending().boxed(), + None => future::ready(Err(redis::RedisError::from(( + redis::ErrorKind::Io, + "unexpected dispatch beyond the scripted replies", + )))) + .boxed(), } } } @@ -855,7 +875,7 @@ mod tests { #[test] fn read_honors_cancellation_before_and_during_the_wait() { - let connection = Scripted::with(vec![]); + let connection = Scripted::hanging(); let adapter = RedisAdapter::new(connection.clone()); let cancelled = ReadContext { timeout_ms: 50, diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index f4ff0ad5..15ab23c2 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -4,8 +4,10 @@ //! //! Without selectors it replays the committed smoke histories. With the //! `DIALCACHE_*_TRACE_DIR` selectors it replays the complete generated corpus -//! and writes the JSONL report named by `DIALCACHE_RUST_REPORT`. Node 24 must -//! be on `PATH` for the coordinator. +//! and writes the JSONL report named by `DIALCACHE_RUST_REPORT`. +//! `DIALCACHE_RUST_SUITE=generated|fixed` runs only the Quint-generated +//! evidence or only the fixed scenarios (the mutation measurement's cohorts). +//! Node 24 must be on `PATH` for the coordinator. #![allow(dead_code)] @@ -99,13 +101,20 @@ fn run() -> Result { failures: 0, coverage_failures: Vec::new(), }; - run.core()?; - run.effects()?; - run.features()?; - run.local_clock()?; - run.scenarios()?; + let suite = run.selection.suite; + if suite.runs_generated() { + run.core()?; + run.effects()?; + run.features()?; + run.local_clock()?; + } + if suite.runs_fixed() { + run.scenarios()?; + } run.protocol_vectors()?; - run.witnesses()?; + if suite.runs_generated() { + run.witnesses()?; + } for (profile, expected) in &run.profile_actions { let selected_full = match profile.as_str() { "core" => run.selection.core.is_directory(), diff --git a/rust/tests/formal/inventory.rs b/rust/tests/formal/inventory.rs index 5e7398ea..edd392ae 100644 --- a/rust/tests/formal/inventory.rs +++ b/rust/tests/formal/inventory.rs @@ -64,6 +64,32 @@ pub enum ProtocolCorpus { Fixed, } +/// Which halves of the suite a run executes (`DIALCACHE_RUST_SUITE`). +/// +/// The mutation measurement runs the Quint-generated evidence and the fixed +/// supplement as separate cohorts; a normal run executes both. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Suite { + /// Generated histories, witness evidence and fixed scenarios (the default and `all`). + All, + /// Generated histories and witness evidence only: no fixed scenarios. + Generated, + /// Fixed scenarios only: no histories and no witness evidence. + Fixed, +} + +impl Suite { + /// Whether generated histories and witness evidence run. + pub fn runs_generated(self) -> bool { + self != Suite::Fixed + } + + /// Whether the fixed scenarios run. + pub fn runs_fixed(self) -> bool { + self != Suite::Generated + } +} + /// The environment-driven selection of what one conformance run replays. #[derive(Debug, Clone, PartialEq, Eq)] pub struct Selection { @@ -79,6 +105,8 @@ pub struct Selection { pub witness_evidence_dir: Option, /// `DIALCACHE_PROTOCOL_CORPUS`. pub protocol_corpus: ProtocolCorpus, + /// `DIALCACHE_RUST_SUITE`. + pub suite: Suite, /// `DIALCACHE_RUST_REPORT`: JSONL assertion report path. pub report: Option, /// `DIALCACHE_BEHAVIOR_SCENARIO`: substring filter on scenario names. @@ -98,6 +126,12 @@ impl Selection { Some("fixed") => ProtocolCorpus::Fixed, Some(other) => return Err(format!("unknown protocol corpus selection {other}")), }; + let suite = match optional("DIALCACHE_RUST_SUITE").as_deref() { + None | Some("all") => Suite::All, + Some("generated") => Suite::Generated, + Some("fixed") => Suite::Fixed, + Some(other) => return Err(format!("unknown suite selection {other}")), + }; Ok(Selection { core: TraceSource::from_env( "DIALCACHE_MBT_TRACE_FILE", @@ -117,6 +151,7 @@ impl Selection { feature_profile: optional("DIALCACHE_FEATURE_PROFILE"), witness_evidence_dir: optional("DIALCACHE_WITNESS_EVIDENCE_DIR").map(PathBuf::from), protocol_corpus, + suite, report: optional("DIALCACHE_RUST_REPORT").map(PathBuf::from), behavior_scenario: optional("DIALCACHE_BEHAVIOR_SCENARIO"), }) diff --git a/rust/tests/metrics_exporters.rs b/rust/tests/metrics_exporters.rs index 71346db7..392cc0e9 100644 --- a/rust/tests/metrics_exporters.rs +++ b/rust/tests/metrics_exporters.rs @@ -15,8 +15,6 @@ use dialcache::{ }; use parking_lot::Mutex; -const NEVER_EXPORTED: &str = "never_export_this_identity"; - fn base() -> Labels { Labels { namespace: Arc::from("logical"), @@ -270,9 +268,10 @@ fn datadog_metric_names_units_and_labels() { assert_eq!(got.value, value, "{kind:?}"); assert_eq!(metric_suffix(kind), suffix); assert_eq!(observer.metric_name(kind), got.name); - for (_, tag) in &got.tags { - assert!(!tag.contains("never_export"), "key leaked to metric labels"); - } + // Every tag is one of the kind's declared labels: no logical key or + // identity ever reaches a metric. + let names: Vec<&str> = got.tags.iter().map(|(name, _)| name.as_str()).collect(); + assert_eq!(names, kind.label_names(), "{kind:?}"); } let recorded = events.lock().clone(); assert_eq!( @@ -608,9 +607,6 @@ mod prometheus_exporter { let mut expected_labels = expected.labels.to_vec(); expected_labels.sort_unstable(); assert_eq!(label_names, expected_labels, "{name}"); - for label in metric.get_label() { - assert!(!label.value().contains(NEVER_EXPORTED)); - } if expected.buckets.is_empty() { assert_eq!(family.type_(), MetricType::COUNTER, "{name}"); assert_eq!(metric.get_counter().value(), 1.0, "{name}"); @@ -724,6 +720,8 @@ mod prometheus_exporter { "failed observer partially registered collectors" ); // ...and the collectors registered before the conflict were rolled + // back (the name stays bound to the DialCache schema, so only that + // schema can reuse it) // back, so their names are free again. let disabled = schemas("") .into_iter() diff --git a/rust/tests/redis_integration.rs b/rust/tests/redis_integration.rs index d7ca251f..c160b640 100644 --- a/rust/tests/redis_integration.rs +++ b/rust/tests/redis_integration.rs @@ -3,19 +3,30 @@ //! Valkey 8 standalone servers plus a single-node Redis 7 cluster, each //! exercised for complete-frame round trips, watermark fencing, the //! `EVALSHA` to `EVAL` recovery after `SCRIPT FLUSH`, and the full -//! invalidation vector corpus (`formal/invalidation_vectors.rs`). +//! invalidation vector corpus (`formal/invalidation_vectors.rs`); and a +//! replicated Redis 7 cluster that pins tracked reads to the slot primary. //! -//! Every test returns early with a printed message unless -//! `DIALCACHE_RUST_INTEGRATION=1` is set; with it set, the `docker` CLI and -//! the images must be available. Containers are removed by a drop guard on -//! every exit path. +//! Every test here is `#[ignore]`d: `cargo test` reports it as ignored, and +//! `make integration-rust` runs the file with `--ignored` where Docker exists. +//! The `docker` CLI and the images must be available. Containers and the +//! cluster network are removed by drop guards on every exit path. //! -//! Cluster coverage is a single primary with every slot assigned: the -//! adapter's explicit primary routing is exercised through a real -//! `ClusterConnection`, but with no replicas present the test cannot -//! distinguish primary from replica routing the way the Go six-node test -//! does (the `redis` crate exposes no dialer hook to remap the internal -//! container addresses a multi-node cluster announces). +//! Cluster coverage has two shapes. `redis_7_single_node_cluster` is one +//! primary owning every slot, announced on loopback, and runs the whole +//! corpus through a real `ClusterConnection`. With no replica present it +//! cannot tell primary routing from replica routing, so +//! `redis_7_replicated_cluster_primary_read` adds one primary plus one +//! replica of the same slots on a dedicated Docker bridge network. The +//! nodes gossip their container addresses, which Docker Desktop does not +//! route from the host; `ClusterClientBuilder::node_address_map` remaps +//! each announced address to the node's published loopback port at connect +//! time (the slot map keeps the announced addresses, so `MOVED` redirects +//! still resolve). With replica reads enabled on the client +//! (`read_routing_strategy(RandomReplicaStrategy)`, the successor of +//! `read_from_replicas`), each node's `INFO commandstats` proves every +//! tracked `MGET` executed on the primary and none on the replica: the +//! property `testPrimaryRead` in the Go test checks with per-node route +//! hooks on a six-node cluster. #![cfg(feature = "redis")] @@ -23,10 +34,11 @@ mod formal; #[path = "formal/invalidation_vectors.rs"] mod invalidation_vectors; +use std::collections::HashMap; use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::process::Command; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use dialcache::protocol::encode_frame; use dialcache::redis::{RedisAdapter, RedisConnection}; @@ -35,21 +47,35 @@ use dialcache::{ ReadResult, Remote, WriteRequest, }; use formal::witness::sha256_hex; -use redis::aio::{ConnectionManager, ConnectionManagerConfig}; -use redis::cluster::ClusterClientBuilder; +use redis::aio::{ConnectionManager, ConnectionManagerConfig, MultiplexedConnection}; +use redis::cluster::{ClusterClientBuilder, NodeAddress}; use redis::cluster_async::ClusterConnection; +use redis::cluster_read_routing::RandomReplicaStrategy; +use redis::cluster_routing::{Route, RoutingInfo, SingleNodeRoutingInfo, Slot, SlotAddr}; use redis::{Client, Value}; -const ENABLE_ENV: &str = "DIALCACHE_RUST_INTEGRATION"; const COMMAND_BUDGET: Duration = Duration::from_secs(2); const READY_BUDGET: Duration = Duration::from_secs(15); -fn enabled(test: &str) -> bool { - if std::env::var(ENABLE_ENV).as_deref() == Ok("1") { - return true; +/// The Go test's server flags: no persistence, no protected mode. +const SERVER_FLAGS: [&str; 6] = ["--save", "", "--appendonly", "no", "--protected-mode", "no"]; +/// Cluster mode without announce overrides: a node announces the address +/// its peers see it from. +const CLUSTER_FLAGS: [&str; 6] = [ + "--cluster-enabled", + "yes", + "--cluster-config-file", + "/tmp/nodes.conf", + "--cluster-node-timeout", + "5000", +]; + +fn server_binary(image: &str) -> &'static str { + if image.contains("valkey") { + "valkey-server" + } else { + "redis-server" } - println!("skipping {test}: set {ENABLE_ENV}=1 to run Docker-backed Redis tests"); - false } fn repo_root() -> PathBuf { @@ -97,11 +123,6 @@ fn free_port() -> u16 { /// For a cluster node the host port is chosen first so the node can announce /// the loopback address a host-side cluster client can reach. fn start_server(image: &str, cluster: bool) -> (Container, String) { - let server = if image.contains("valkey") { - "valkey-server" - } else { - "redis-server" - }; let announce_port = cluster.then(free_port); let publish = match announce_port { Some(port) => format!("127.0.0.1:{port}:6379"), @@ -115,22 +136,12 @@ fn start_server(image: &str, cluster: bool) -> (Container, String) { "-p", &publish, image, - server, - "--save", - "", - "--appendonly", - "no", - "--protected-mode", - "no", + server_binary(image), ]; + args.extend(SERVER_FLAGS); if let Some(port) = &announce { + args.extend(CLUSTER_FLAGS); args.extend([ - "--cluster-enabled", - "yes", - "--cluster-config-file", - "/tmp/nodes.conf", - "--cluster-node-timeout", - "5000", "--cluster-announce-ip", "127.0.0.1", "--cluster-announce-port", @@ -139,13 +150,34 @@ fn start_server(image: &str, cluster: bool) -> (Container, String) { } let id = docker(&args).expect("start container"); let container = Container { id }; - let endpoint = docker(&["port", &container.id, "6379/tcp"]) + let endpoint = published_endpoint(&container); + (container, endpoint) +} + +/// The `host:port` Docker published for the container's Redis port. +fn published_endpoint(container: &Container) -> String { + docker(&["port", &container.id, "6379/tcp"]) .expect("published port") .lines() .next() .expect("one published endpoint") - .to_string(); - (container, endpoint) + .to_string() +} + +/// The value of `name` in an `INFO` or `CLUSTER INFO` reply, or empty when +/// the field is absent. Lines keep their `\r`, which `trim` removes. +fn info_field(info: &str, name: &str) -> String { + info.lines() + .find_map(|line| line.trim().strip_prefix(name)?.strip_prefix(':')) + .unwrap_or("") + .to_string() +} + +/// A node's `CLUSTER INFO` reports a healthy cluster with every slot served. +fn cluster_ready(info: &str) -> bool { + info_field(info, "cluster_state") == "ok" + && info_field(info, "cluster_slots_assigned") == "16384" + && info_field(info, "cluster_slots_ok") == "16384" } async fn wait_for_ping(endpoint: &str) { @@ -200,16 +232,7 @@ async fn single_node_cluster(image: &str) -> (Container, ClusterConnection) { loop { let info = docker(&["exec", &container.id, "redis-cli", "CLUSTER", "INFO"]).unwrap_or_default(); - let field = |name: &str| { - info.lines() - .find_map(|line| line.trim().strip_prefix(name)?.strip_prefix(':')) - .unwrap_or("") - .to_string() - }; - if field("cluster_state") == "ok" - && field("cluster_slots_assigned") == "16384" - && field("cluster_slots_ok") == "16384" - { + if cluster_ready(&info) { break; } assert!( @@ -448,9 +471,6 @@ async fn exercise(connection: C, label: &str) { } fn run_standalone(test: &str, image: &str) { - if !enabled(test) { - return; - } let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); runtime.block_on(async { let (_container, connection) = standalone(image).await; @@ -459,28 +479,425 @@ fn run_standalone(test: &str, image: &str) { } #[test] +#[ignore = "Docker-backed; run through make integration-rust"] fn redis_6_2_standalone() { run_standalone("redis6.2", "redis:6.2-alpine"); } #[test] +#[ignore = "Docker-backed; run through make integration-rust"] fn redis_7_standalone() { run_standalone("redis7", "redis:7-alpine"); } #[test] +#[ignore = "Docker-backed; run through make integration-rust"] fn valkey_8_standalone() { run_standalone("valkey8", "valkey/valkey:8-alpine"); } #[test] +#[ignore = "Docker-backed; run through make integration-rust"] fn redis_7_single_node_cluster() { - if !enabled("cluster") { - return; - } let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); runtime.block_on(async { let (_container, connection) = single_node_cluster("redis:7-alpine").await; exercise(connection, "cluster").await; }); } + +/// A Docker bridge network, removed when dropped. The daemon detaches a +/// force-removed container's endpoint asynchronously, so removal retries +/// while the network still reports active endpoints. +struct Network { + name: String, +} + +impl Drop for Network { + fn drop(&mut self) { + let deadline = Instant::now() + Duration::from_secs(10); + loop { + let output = Command::new("docker") + .args(["network", "rm", &self.name]) + .output(); + let retry = match &output { + Ok(output) if output.status.success() => false, + Ok(output) => String::from_utf8_lossy(&output.stderr).contains("active endpoints"), + Err(_) => false, + }; + if !retry || Instant::now() >= deadline { + break; + } + std::thread::sleep(Duration::from_millis(100)); + } + } +} + +/// One node of the replicated cluster: its container, the loopback +/// endpoint Docker published for host clients, and the bridge address the +/// node announces to its peers (and therefore in `CLUSTER SLOTS`). +struct ClusterNode { + container: Container, + endpoint: String, + bridge_ip: String, +} + +impl ClusterNode { + fn host_port(&self) -> u16 { + self.endpoint + .rsplit(':') + .next() + .and_then(|port| port.parse().ok()) + .expect("published endpoint has a port") + } + + /// Run `redis-cli` inside the container for cluster setup commands. + fn redis_cli(&self, args: &[&str]) -> Result { + let mut command = vec!["exec", self.container.id.as_str(), "redis-cli"]; + command.extend(args); + docker(&command) + } +} + +/// One primary owning every slot and one replica of it, gossiping over a +/// dedicated bridge network. Field order removes the containers before the +/// network they are attached to. +struct ReplicatedCluster { + primary: ClusterNode, + replica: ClusterNode, + _network: Network, +} + +/// `docker run` a cluster node on `network`, publishing its port on a free +/// loopback port and letting it announce its bridge address to peers. +fn start_cluster_node(image: &str, network: &str) -> ClusterNode { + let mut args = vec![ + "run", + "-d", + "--rm", + "--network", + network, + "-p", + "127.0.0.1::6379", + image, + server_binary(image), + ]; + args.extend(SERVER_FLAGS); + args.extend(CLUSTER_FLAGS); + let id = docker(&args).expect("start cluster node"); + let container = Container { id }; + let endpoint = published_endpoint(&container); + let bridge_ip = docker(&[ + "inspect", + "--format", + "{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + &container.id, + ]) + .expect("bridge address"); + assert!( + !bridge_ip.is_empty(), + "container {} has no bridge address", + container.id + ); + ClusterNode { + container, + endpoint, + bridge_ip, + } +} + +/// Bring up the replicated cluster: assign every slot to the primary, have +/// it `MEET` the replica, wait for gossip to teach the replica the primary's +/// id, `REPLICATE`, then wait until both nodes report `cluster_state:ok` and +/// the replica's link to its primary is up. +async fn replicated_cluster(image: &str) -> ReplicatedCluster { + let nanos = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|since| since.as_nanos()) + .unwrap_or_default(); + let name = format!("dialcache-rust-{}-{nanos}", std::process::id()); + docker(&["network", "create", &name]).expect("create network"); + let network = Network { name: name.clone() }; + let primary = start_cluster_node(image, &name); + let replica = start_cluster_node(image, &name); + let cluster = ReplicatedCluster { + primary, + replica, + _network: network, + }; + wait_for_ping(&cluster.primary.endpoint).await; + wait_for_ping(&cluster.replica.endpoint).await; + cluster + .primary + .redis_cli(&["CLUSTER", "ADDSLOTSRANGE", "0", "16383"]) + .expect("assign every slot"); + cluster + .primary + .redis_cli(&["CLUSTER", "MEET", &cluster.replica.bridge_ip, "6379"]) + .expect("meet the replica"); + let primary_id = cluster + .primary + .redis_cli(&["CLUSTER", "MYID"]) + .expect("primary id"); + + // CLUSTER REPLICATE rejects an id the replica has not yet learned. + let deadline = Instant::now() + READY_BUDGET; + loop { + let nodes = cluster + .replica + .redis_cli(&["CLUSTER", "NODES"]) + .unwrap_or_default(); + if nodes.lines().any(|line| line.starts_with(&primary_id)) { + break; + } + assert!( + Instant::now() < deadline, + "replica never learned the primary {primary_id}:\n{nodes}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + cluster + .replica + .redis_cli(&["CLUSTER", "REPLICATE", &primary_id]) + .expect("replicate the primary"); + + // The client seeds its slot map from the primary's CLUSTER SLOTS, which + // omits a replica until gossip has reported a non-zero replication + // offset for it; wait for that too, or the client would never learn the + // replica exists. + let deadline = Instant::now() + READY_BUDGET; + loop { + let primary_info = cluster + .primary + .redis_cli(&["CLUSTER", "INFO"]) + .unwrap_or_default(); + let replica_info = cluster + .replica + .redis_cli(&["CLUSTER", "INFO"]) + .unwrap_or_default(); + let replication = cluster + .replica + .redis_cli(&["INFO", "replication"]) + .unwrap_or_default(); + let slots = cluster + .primary + .redis_cli(&["CLUSTER", "SLOTS"]) + .unwrap_or_default(); + if cluster_ready(&primary_info) + && cluster_ready(&replica_info) + && info_field(&replication, "role") == "slave" + && info_field(&replication, "master_link_status") == "up" + && slots + .lines() + .any(|line| line.trim() == cluster.replica.bridge_ip) + { + break; + } + assert!( + Instant::now() < deadline, + "replicated cluster did not become ready:\nprimary:\n{primary_info}\nreplica:\n{replica_info}\nreplication:\n{replication}\nslots:\n{slots}" + ); + tokio::time::sleep(Duration::from_millis(50)).await; + } + cluster +} + +/// A cluster connection that prefers replicas for reads +/// (`RandomReplicaStrategy`, what the deprecated `read_from_replicas` +/// installs). The nodes announce their bridge addresses, which the host +/// cannot route to under Docker Desktop; `node_address_map` redirects each +/// to the node's published loopback port when the client connects. +async fn replica_reading_connection(cluster: &ReplicatedCluster) -> ClusterConnection { + let mut addresses = HashMap::new(); + for node in [&cluster.primary, &cluster.replica] { + addresses.insert( + NodeAddress::new(node.bridge_ip.as_str(), 6379), + NodeAddress::new("127.0.0.1", node.host_port()), + ); + } + ClusterClientBuilder::new([format!("redis://{}", cluster.primary.endpoint)]) + .connection_timeout(COMMAND_BUDGET) + .response_timeout(COMMAND_BUDGET) + .read_routing_strategy(RandomReplicaStrategy) + .node_address_map(addresses) + .build() + .expect("cluster client") + .get_async_connection() + .await + .expect("cluster connection") +} + +/// A plain connection to one node's published endpoint, bypassing cluster +/// routing, for per-node observation. +async fn direct_connection(endpoint: &str) -> MultiplexedConnection { + Client::open(format!("redis://{endpoint}")) + .expect("client url") + .get_multiplexed_async_connection() + .await + .expect("direct connection") +} + +/// How many times the node executed `command`, from `INFO commandstats` +/// (`cmdstat_:calls=N,...`). Commands the node refused with +/// `MOVED` count under `rejected_calls`, not `calls`; a command the node +/// never ran has no line at all. +async fn executed_calls(connection: &mut MultiplexedConnection, command: &str) -> u64 { + let info: String = redis::cmd("INFO") + .arg("commandstats") + .query_async(connection) + .await + .expect("INFO commandstats"); + info_field(&info, &format!("cmdstat_{command}")) + .split(',') + .find_map(|field| field.strip_prefix("calls=")) + .and_then(|calls| calls.parse().ok()) + .unwrap_or(0) +} + +/// Route one `GET key` to a replica of the key's slot through the cluster +/// connection. `ReplicaRequired` can only choose a replica when the client's +/// slot map lists one and otherwise falls back to the primary, so this +/// executing on the replica proves the client knows the replica and reached +/// it through the address map: a `ReplicaOptional` route would have had a +/// replica to pick, which makes the tracked-read assertion decisive. +async fn replica_routed_get(connection: &ClusterConnection, key: &str) { + let route = Route::with_slot(Slot::for_key(key), SlotAddr::ReplicaRequired); + let routing = RoutingInfo::SingleNode(SingleNodeRoutingInfo::SpecificNode(route)); + let mut get = redis::cmd("GET"); + get.arg(key); + connection + .clone() + .route_command(get, routing) + .await + .expect("replica-routed GET"); +} + +/// Poll the replica directly (after `READONLY`, so it serves rather than +/// redirects) until the exact frame bytes have replicated. +async fn wait_for_replica_frame(replica: &mut MultiplexedConnection, key: &str, frame: &Frame) { + redis::cmd("READONLY") + .query_async::<()>(replica) + .await + .expect("READONLY"); + let encoded = encode_frame(frame).expect("encode frame"); + let deadline = Instant::now() + READY_BUDGET; + loop { + let stored: Option> = redis::cmd("GET") + .arg(key) + .query_async(replica) + .await + .expect("GET on the replica"); + if stored.as_deref() == Some(encoded.as_slice()) { + return; + } + assert!( + Instant::now() < deadline, + "replica never received {key}: {stored:?}" + ); + tokio::time::sleep(Duration::from_millis(10)).await; + } +} + +/// `go/redis_integration_test.go` `testPrimaryRead`'s routing assertion on +/// a cluster that has a replica: with replica reads enabled on the client, +/// every tracked `MGET` still executes on the slot primary and none on the +/// replica, so a lagging replica can never hide a watermark. Each node's own +/// `INFO commandstats` is the witness. +/// +/// The `redis` crate's `RandomReplicaStrategy` picks uniformly among +/// replicas only, so a `ReplicaOptional` route would send every one of these +/// reads to the replica and fail on the first. Twelve reads keep the check +/// decisive even against a strategy that chose uniformly between both +/// nodes, where all twelve landing on the primary by chance would be 2^-12, +/// about 0.02%. +#[test] +#[ignore = "Docker-backed; run through make integration-rust"] +fn redis_7_replicated_cluster_primary_read() { + const TRACKED_READS: u64 = 12; + const UNTRACKED_READS: u64 = 3; + let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); + runtime.block_on(async { + let cluster = replicated_cluster("redis:7-alpine").await; + let adapter = RedisAdapter::new(replica_reading_connection(&cluster).await); + let mut primary = direct_connection(&cluster.primary.endpoint).await; + let mut replica = direct_connection(&cluster.replica.endpoint).await; + + let key = "{primary-rust-replicated}:value"; + let watermark = "{primary-rust-replicated}:watermark"; + let frame = Frame { + created_at_ms: 2_000, + payload: Payload::binary(vec![0, 1, 255]), + }; + adapter + .write(WriteRequest { + value_key: key.to_string(), + frame: frame.clone(), + ttl_ms: 60_000, + }) + .await + .expect("tracked write"); + wait_for_replica_frame(&mut replica, key, &frame).await; + + // Precondition: the client's slot map holds the replica, so replica + // routing is a real choice the adapter is declining. + let primary_gets = executed_calls(&mut primary, "get").await; + let replica_gets = executed_calls(&mut replica, "get").await; + replica_routed_get(adapter.connection(), key).await; + assert_eq!( + ( + executed_calls(&mut primary, "get").await - primary_gets, + executed_calls(&mut replica, "get").await - replica_gets + ), + (0, 1), + "replica-routed GET executions (primary, replica): the client does not know the replica" + ); + + let primary_before = executed_calls(&mut primary, "mget").await; + let replica_before = executed_calls(&mut replica, "mget").await; + for read in 0..TRACKED_READS { + let got = adapter + .read(tracked(key, watermark), context()) + .await + .expect("tracked read"); + assert_eq!(got, ReadResult::Hit(frame.clone()), "tracked read {read}"); + } + let primary_after = executed_calls(&mut primary, "mget").await; + let replica_after = executed_calls(&mut replica, "mget").await; + assert_eq!( + ( + primary_after - primary_before, + replica_after - replica_before + ), + (TRACKED_READS, 0), + "tracked MGET executions (primary, replica): primary {primary_before} -> \ + {primary_after}, replica {replica_before} -> {replica_after}" + ); + + // Untracked reads keep the client's own routing and may execute on + // either node with replica reads enabled; only the total is pinned + // and the split is reported. + let primary_gets = executed_calls(&mut primary, "get").await; + let replica_gets = executed_calls(&mut replica, "get").await; + for read in 0..UNTRACKED_READS { + let got = adapter + .read( + ReadRequest { + value_key: key.to_string(), + watermark_key: None, + }, + context(), + ) + .await + .expect("untracked read"); + assert_eq!(got, ReadResult::Hit(frame.clone()), "untracked read {read}"); + } + let primary_delta = executed_calls(&mut primary, "get").await - primary_gets; + let replica_delta = executed_calls(&mut replica, "get").await - replica_gets; + assert_eq!( + primary_delta + replica_delta, + UNTRACKED_READS, + "untracked GET executions: primary {primary_delta}, replica {replica_delta}" + ); + println!("replicated: untracked GETs executed on primary={primary_delta} replica={replica_delta}"); + }); +} diff --git a/rust/tests/tokio_runtime.rs b/rust/tests/tokio_runtime.rs index f31574dd..2acaed60 100644 --- a/rust/tests/tokio_runtime.rs +++ b/rust/tests/tokio_runtime.rs @@ -93,23 +93,38 @@ async fn a_source_deadline_returns_a_timeout_and_leaves_the_source_running() { .build() .expect("explicit handle"); let calls = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Notify::new()); + let settled = Arc::new(tokio::sync::Notify::new()); let request = cache.enable_guard(); + // The source waits for an explicit release, so the deadline result never + // races a second timer on a slow runner. + let load = { + let (calls, release, settled) = (calls.clone(), release.clone(), settled.clone()); + move |_: Scope| { + let (calls, release, settled) = (calls.clone(), release.clone(), settled.clone()); + Box::pin(async move { + release.notified().await; + calls.fetch_add(1, Ordering::SeqCst); + settled.notify_one(); + Ok::(1) + }) as futures::future::BoxFuture<'static, Result> + } + }; let error = cache .get_or_load( request.scope(), operation("slow").budget(SourceBudget::Millis(20)), - source(&calls, Duration::from_millis(150), 1), + load, ) .await .expect_err("deadline"); assert!(error.is_fallback_timeout(), "unexpected error: {error}"); assert_eq!(calls.load(Ordering::SeqCst), 0); - tokio::time::sleep(Duration::from_millis(300)).await; - assert_eq!( - calls.load(Ordering::SeqCst), - 1, - "the deadline cancelled the source" - ); + release.notify_one(); + tokio::time::timeout(Duration::from_secs(5), settled.notified()) + .await + .expect("the deadline cancelled the source"); + assert_eq!(calls.load(Ordering::SeqCst), 1); } #[tokio::test] diff --git a/test/formal-mutation-shards.test.ts b/test/formal-mutation-shards.test.ts index b552f131..30eb38ce 100644 --- a/test/formal-mutation-shards.test.ts +++ b/test/formal-mutation-shards.test.ts @@ -19,12 +19,12 @@ const shared = await import(new URL("../formal/mutation-reports.mjs", import.met shardFromArguments(argv: string[]): Shard; partitionMutations(mutations: T[], shard: Shard): T[]; shardDirectory(output: string, shard: Shard): string; - fingerprintFiles(directory: string, paths: string[]): { files: number; sha256: string }; + fingerprintFiles(directory: string, paths: string[], options?: { exclude?: string[] }): { files: number; sha256: string }; requiredDetectionRegressions(entries: CatalogEntry[], results: Mutation[]): string[]; goDetection(mutations: Mutation[]): Record; typescriptDetection(mutations: Mutation[], cases: { id: string; vectors: unknown[] }[]): Record; gateDetections(language: Language, report: Report, entries: CatalogEntry[], options?: { directory?: string; summarize?: boolean }): void; - languages: { ts: Language; go: Language }; + languages: { ts: Language; go: Language; rust: Language & { exclude: string[] } }; }; const merge = await import(new URL("../formal/merge-mutation-reports.mjs", import.meta.url).href) as { canonical(value: unknown): string; @@ -414,13 +414,52 @@ describe("mutation shard merge over a shard directory", () => { writeShards(lost, join(directory, ".formal-traces/go-semantic/shards")); expect(() => mergeMutationReports("go", { directory })).toThrow(/Lost required detections: M11\/generated/); expect(readReport(".formal-traces/go-semantic/report.json")).toMatchObject({ complete: false, requiredDetectionRegressions: ["M11/generated"], error: expect.stringContaining("M11/generated") }); - expect(() => mergeMutationReports("rust", { directory })).toThrow(/Expected language ts or go/); + expect(() => mergeMutationReports("zig", { directory })).toThrow(/Expected language ts, go or rust/); expect(() => mergeMutationReports("go", { directory, shardsDirectory: "nowhere" })).toThrow(/no shard directory/); expect(readReport(".formal-traces/go-semantic/report.json")).toMatchObject({ complete: false, error: expect.stringContaining("no shard directory") }); // The command line rejects a bad language before touching any report directory. const { spawnSync } = await import("node:child_process"); - const usage = spawnSync(process.execPath, [new URL("../formal/merge-mutation-reports.mjs", import.meta.url).pathname, "rust"], { encoding: "utf8" }); + const usage = spawnSync(process.execPath, [new URL("../formal/merge-mutation-reports.mjs", import.meta.url).pathname, "zig"], { encoding: "utf8" }); expect(usage.status).toBe(2); - expect(usage.stderr).toMatch(/Usage: node formal\/merge-mutation-reports.mjs /); + expect(usage.stderr).toMatch(/Usage: node formal\/merge-mutation-reports.mjs /); }); }); + +describe("Rust mutation language", () => { + let directory: string; + beforeEach(() => { directory = mkdtempSync(join(tmpdir(), "dialcache-rust-mutation-language-")); }); + afterEach(() => rmSync(directory, { recursive: true, force: true })); + const put = (path: string, text: string) => { mkdirSync(join(directory, path, ".."), { recursive: true }); writeFileSync(join(directory, path), text); }; + + it("fingerprints the crate sources and files without the build directory", () => { + for (const path of ["formal/rust-mutations.json", "test/a.test.ts", "src/a.ts", "go/redis_adapter.go", "rust/Cargo.toml", "rust/Cargo.lock", "rust/src/lib.rs", "rust/tests/conformance.rs"]) put(path, path); + const clean = fingerprintFiles(directory, languages.rust.inputs, { exclude: languages.rust.exclude }); + put("rust/target/release/deps/libdialcache.rlib", "build output"); + put("rust/target/semantic/report.json", "{}"); + expect(fingerprintFiles(directory, languages.rust.inputs, { exclude: languages.rust.exclude })).toEqual(clean); + expect(clean.files).toBe(8); + // Without the exclusion the build output would count, so the exclusion is what keeps a checkout and its workspace copy equal. + expect(fingerprintFiles(directory, languages.rust.inputs).files).toBe(10); + expect(fingerprintFiles(directory, ["rust/Cargo.toml"])).toEqual(fingerprintFiles(directory, ["rust/Cargo.toml"], { exclude: ["rust/target"] })); + expect(fingerprintFiles(directory, ["rust/Cargo.toml"]).files).toBe(1); + }); + + it("merges Rust shards with the same four-cohort strictness as Go and renders the Rust report", () => { + const catalogSha256 = sha256(readRepo("formal/go-mutations.json")); + const inputs = { files: 3, sha256: "dd".repeat(32) }; + const single = { ...goSingleReport(catalogSha256, inputs), cargo: "cargo 1.98.1 (797e8a9bc 2026-08-05)" } as Report; + delete (single as Record).go; + const merged = mergeShardReports(languages.rust, goShardReports(single, 3), { catalog: goCatalog, catalogSha256, inputs }); + gateDetections(languages.rust, merged, goCatalog.mutations); + expect(merged.complete).toBe(true); + expect(merged.detection).toEqual(goDetection(single.mutations)); + expect(merged.requiredDetectionRegressions).toEqual([]); + expect(languages.rust.output).toBe(".formal-traces/rust-semantic"); + expect(languages.rust.catalog).toBe("formal/rust-mutations.json"); + const markdown = languages.rust.markdown(merged); + expect(markdown).toContain("# Rust semantic mutation measurement"); + expect(markdown).toContain("| M01 | C45.maximum-age-exclusive | survived | detected | detected | detected |"); + expect(markdown).toContain("Merged from 3 shards"); + }); +}); + diff --git a/test/formal-rust-replay.test.ts b/test/formal-rust-replay.test.ts index 277f3fe2..ca9a87ce 100644 --- a/test/formal-rust-replay.test.ts +++ b/test/formal-rust-replay.test.ts @@ -165,11 +165,11 @@ describe("Rust validation lanes", () => { expect(smoke.filter(step => step.command === "cargo")).toHaveLength(1); }); - it("runs the real-server integration binary only through the integration lane, opted in by its environment variable", () => { + it("runs the real-server integration binary only through the integration lane, which selects its ignored tests", () => { const lane = validationPlan("integration-rust", { directory }); - expect(lane).toEqual([{ label: "Run Rust Redis/Valkey/Cluster integrations", command: "cargo", args: ["test", "--all-features", "--test", "redis_integration"], cwd: "rust", env: { DIALCACHE_RUST_INTEGRATION: "1" } }]); + expect(lane).toEqual([{ label: "Run Rust Redis/Valkey/Cluster integrations", command: "cargo", args: ["test", "--all-features", "--test", "redis_integration", "--", "--ignored"], cwd: "rust" }]); expect(validationPlan("integration", { directory })).toEqual(["integration-ts", "integration-go", "integration-rust"].flatMap(target => validationPlan(target, { directory }))); - for (const target of ["check-rust", "smoke", "formal-rust"]) expect(validationPlan(target, { directory }).some(step => step.env?.DIALCACHE_RUST_INTEGRATION), target).toBe(false); + for (const target of ["check-rust", "smoke", "formal-rust"]) expect(validationPlan(target, { directory }).some(step => step.args?.includes("--ignored")), target).toBe(false); }); it("probes the pinned cargo exactly for the Rust lanes", async () => { @@ -189,8 +189,8 @@ describe("Rust validation lanes", () => { const environment = { ...process.env, PATH: `${join(temporary, "bin")}${delimiter}${process.env.PATH ?? ""}` }; const options = { directory: temporary, environment, nodeVersion: "v24.20.0" }; tool("cargo", 'console.error("cargo: command not found"); process.exit(127)'); - for (const target of ["check-rust", "formal-rust", "smoke", "check", "integration-rust"]) expect(() => checkPrerequisites(target, options), target).toThrow(/Cannot run cargo/); - for (const target of ["check-ts", "check-go", "formal-ts", "formal-go", "mutations", "audit"]) expect(() => checkPrerequisites(target, options), target).not.toThrow(); + for (const target of ["check-rust", "formal-rust", "smoke", "check", "integration-rust", "mutations-rust", "mutations", "explore"]) expect(() => checkPrerequisites(target, options), target).toThrow(/Cannot run cargo/); + for (const target of ["check-ts", "check-go", "formal-ts", "formal-go", "mutations-ts", "mutations-go", "mutations-merge-rust", "audit"]) expect(() => checkPrerequisites(target, options), target).not.toThrow(); tool("cargo", 'console.log("cargo 1.97.0 (abcdef 2026-06-01)")'); expect(() => checkPrerequisites("check-rust", options)).toThrow(/requires cargo 1\.98\.1; found cargo 1\.97\.0/); tool("cargo", 'console.log("cargo 1.98.10 (abcdef 2026-06-01)")'); diff --git a/test/formal-rust-semantic-runner.test.ts b/test/formal-rust-semantic-runner.test.ts new file mode 100644 index 00000000..6e57d7d6 --- /dev/null +++ b/test/formal-rust-semantic-runner.test.ts @@ -0,0 +1,123 @@ +import { readFileSync } from "node:fs"; +import { describe, expect, it } from "vitest"; + +const moduleUrl = new URL("../formal/measure-rust-semantics.mjs", import.meta.url).href; +type Cohort = { state: string; passed: number; failed: number; failingTests: string[]; executedTests: string[] }; +const { evaluateCargoTestOutput, evaluateRustReport, infrastructureTestFile } = await import(moduleUrl) as { + evaluateCargoTestOutput(output: string, exitCode: number, expectedBinaries?: number): Cohort; + evaluateRustReport(text: string, exitCode: number): Cohort; + infrastructureTestFile: RegExp; +}; + +// libtest output as `cargo test --release --no-fail-fast --lib --test tokio_runtime` prints it. +function cargoOutput(libOutcome: string, integrationOutcome: string): string { + const failed = (outcome: string) => (outcome === "FAILED" ? 1 : 0); + return [ + " Running unittests src/lib.rs (/tmp/target/release/deps/dialcache-0123)", + "", + "running 2 tests", + `test policy::tests::defaults ... ${libOutcome}`, + "test codec::tests::round_trip ... ok", + "", + `test result: ${failed(libOutcome) ? "FAILED" : "ok"}. ${2 - failed(libOutcome)} passed; ${failed(libOutcome)} failed; 0 ignored; 0 measured; 0 filtered out; finished in 0.01s`, + "", + " Running tests/tokio_runtime.rs (/tmp/target/release/deps/tokio_runtime-4567)", + "", + "running 2 tests", + `test concurrent_callers_coalesce ... ${integrationOutcome}`, + "test policy::tests::defaults ... ignored", + "", + `test result: ${failed(integrationOutcome) ? "FAILED" : "ok"}. ${1 - failed(integrationOutcome)} passed; ${failed(integrationOutcome)} failed; 1 ignored; 0 measured; 0 filtered out; finished in 0.30s`, + "", + ].join("\n"); +} + +describe("Rust ordinary cohort evaluation", () => { + it("qualifies test names by binary, drops ignored tests and survives a clean run", () => { + const cohort = evaluateCargoTestOutput(cargoOutput("ok", "ok"), 0, 2); + expect(cohort).toMatchObject({ state: "survived", passed: 3, failed: 0, failingTests: [] }); + expect(cohort.executedTests).toEqual(["src/lib.rs::policy::tests::defaults", "src/lib.rs::codec::tests::round_trip", "tests/tokio_runtime.rs::concurrent_callers_coalesce"]); + }); + + it("detects a failing test in either binary and names it", () => { + expect(evaluateCargoTestOutput(cargoOutput("FAILED", "ok"), 101, 2)).toMatchObject({ state: "detected", passed: 2, failed: 1, failingTests: ["src/lib.rs::policy::tests::defaults"] }); + expect(evaluateCargoTestOutput(cargoOutput("ok", "FAILED"), 101, 2)).toMatchObject({ state: "detected", failingTests: ["tests/tokio_runtime.rs::concurrent_callers_coalesce"] }); + }); + + it("treats a missing binary, a missing result line or an exit code that disagrees with the outcomes as infrastructure failure", () => { + expect(() => evaluateCargoTestOutput(cargoOutput("ok", "ok"), 0, 3)).toThrow(/expected 3 test binaries/); + expect(() => evaluateCargoTestOutput("running 1 test\ntest a ... ok\n", 0)).toThrow(/no libtest result line/); + expect(() => evaluateCargoTestOutput(cargoOutput("ok", "ok"), 101, 2)).toThrow(/without a failing test/); + expect(() => evaluateCargoTestOutput(cargoOutput("FAILED", "ok"), 0, 2)).toThrow(/exited 0 with failing tests/); + }); + + it("excludes the harness, its controls, infrastructure, protocol vector suites and real servers from ordinary tests", () => { + for (const file of ["conformance.rs", "settlement_control.rs", "harness_infra.rs", "redis_integration.rs", "protocol_keys.rs", "protocol_frames.rs"]) expect(infrastructureTestFile.test(file), file).toBe(true); + for (const file of ["metrics_exporters.rs", "tokio_runtime.rs", "policy_helpers.rs"]) expect(infrastructureTestFile.test(file), file).toBe(false); + }); +}); + +function report(cases: [string, string][], finish = true, extra: Record = {}): string { + const failed = cases.filter(([, status]) => status === "failed").length; + const lines = [ + JSON.stringify({ schemaVersion: 1, kind: "start", implementation: "rust", startedAt: 1 }), + ...cases.map(([id, status]) => JSON.stringify({ kind: "case", id, status, startedAt: 2, finishedAt: 3, ...(status === "failed" ? { message: "expected 1 got 2" } : {}) })), + ]; + if (finish) lines.push(JSON.stringify({ kind: "finish", status: failed ? "failed" : "passed", finishedAt: 4, cases: cases.length, failed, ...extra })); + return lines.join("\n") + "\n"; +} + +describe("Rust harness report evaluation", () => { + it("survives a complete passing report and lists every executed case", () => { + const cohort = evaluateRustReport(report([["sampled/core/0", "passed"], ["scenario/read/first-fill", "passed"]]), 0); + expect(cohort).toMatchObject({ state: "survived", passed: 2, failed: 0, failingTests: [], executedTests: ["sampled/core/0", "scenario/read/first-fill"] }); + }); + + it("detects a failed case only when the harness also exited nonzero", () => { + const text = report([["sampled/core/0", "failed"], ["scenario/read/first-fill", "passed"]]); + expect(evaluateRustReport(text, 101)).toMatchObject({ state: "detected", passed: 1, failed: 1, failingTests: ["sampled/core/0"] }); + expect(() => evaluateRustReport(text, 0)).toThrow(/exited 0 with failed cases/); + expect(() => evaluateRustReport(report([["sampled/core/0", "passed"]]), 101)).toThrow(/without a failed case/); + }); + + it("rejects crashed, empty, duplicated or inconsistent reports instead of crediting them", () => { + expect(() => evaluateRustReport("", 0)).toThrow(/empty Rust harness report/); + expect(() => evaluateRustReport(report([["sampled/core/0", "passed"]], false), 0)).toThrow(/no finish record/); + expect(() => evaluateRustReport(report([["sampled/core/0", "passed"], ["sampled/core/0", "passed"]]), 0)).toThrow(/duplicate case/); + expect(() => evaluateRustReport(report([], true), 0)).toThrow(/no cases/); + expect(() => evaluateRustReport(report([["sampled/core/0", "passed"]], true, { cases: 2 }), 0)).toThrow(/finish totals disagree/); + expect(() => evaluateRustReport(report([["sampled/core/0", "passed"]], true, { status: "failed" }), 0)).toThrow(/finish status disagrees/); + }); +}); + +describe("Rust fault catalog", () => { + const readRepo = (path: string) => readFileSync(new URL(`../${path}`, import.meta.url), "utf8"); + type Entry = { id: string; case: string; description: string; typescriptMutation: string; edits: { path: string; before: string; after: string }[]; requiredDetections: string[]; typescriptRequiredDetections: string[] }; + const catalog = JSON.parse(readRepo("formal/rust-mutations.json")) as { schemaVersion: number; mutations: Entry[] }; + const typescript = JSON.parse(readRepo("formal/semantic-mutations.json")) as { mutations: { id: string; case: string; description: string; requiredDetections: string[] }[] }; + + it("names every TypeScript fault once with the same case and description", () => { + expect(catalog.schemaVersion).toBe(1); + expect(catalog.mutations.map(m => m.id)).toEqual(typescript.mutations.map(m => m.id)); + for (const entry of catalog.mutations) { + const counterpart = typescript.mutations.find(m => m.id === entry.typescriptMutation)!; + expect(counterpart, entry.id).toBeDefined(); + expect(entry.case, entry.id).toBe(counterpart.case); + expect(entry.description, entry.id).toBe(counterpart.description); + expect(entry.typescriptRequiredDetections, entry.id).toEqual(counterpart.requiredDetections); + expect(entry.requiredDetections, entry.id).toEqual(expect.arrayContaining(["generated", "portable"])); + expect(entry.requiredDetections.every(cohort => ["generated", "fixed", "portable"].includes(cohort)), entry.id).toBe(true); + } + }); + + it("anchors every edit exactly once in the crate's production sources", () => { + for (const entry of catalog.mutations) { + expect(entry.edits.length, entry.id).toBeGreaterThan(0); + for (const edit of entry.edits) { + expect(edit.path, entry.id).toMatch(/^rust\/src\/[\w/-]+\.rs$/); + expect(edit.before, entry.id).not.toBe(edit.after); + expect(readRepo(edit.path).split(edit.before).length, `${entry.id} ${edit.path}`).toBe(2); + } + } + }); +}); diff --git a/test/formal-validation.test.ts b/test/formal-validation.test.ts index c9d7c69c..473dae0f 100644 --- a/test/formal-validation.test.ts +++ b/test/formal-validation.test.ts @@ -177,18 +177,18 @@ process.exit(Number(process.argv[3] ?? 0));\n`); expect(ts.some(step => step.args?.[0] === "formal/witnesses.mjs" || step.args?.[0] === "formal/run-models.mjs")).toBe(false); }); - it("lets Go parity and both mutation measurements run off the generated corpus without completion checks", () => { + it("lets Go parity and every mutation measurement run off the generated corpus without completion checks", () => { const isCompletionCheck = (step: Step) => step.args?.[0] === "formal/conformance.mjs" && step.args[1] === "check"; const go = validationPlan("formal-go", { directory }); expect(go[0]!.remove).toEqual([".formal-traces/go-completion.json"]); expect(go.some(step => step.args?.some(argument => argument.startsWith(".formal-traces/ts-")))).toBe(false); expect(go.filter(isCompletionCheck).map(step => step.args)).toEqual([["formal/conformance.mjs", "check", ".formal-traces/go-completion.json", ".formal-traces/go-context.json"]]); expect(go.some(step => step.args?.[0] === "formal/witnesses.mjs" || step.args?.[0] === "formal/run-models.mjs")).toBe(false); - for (const [target, script] of [["mutations-ts", "formal/measure-semantics.mjs"], ["mutations-go", "formal/measure-go-semantics.mjs"]] as const) { + for (const [target, script] of [["mutations-ts", "formal/measure-semantics.mjs"], ["mutations-go", "formal/measure-go-semantics.mjs"], ["mutations-rust", "formal/measure-rust-semantics.mjs"]] as const) { expect(validationPlan(target, { directory }).map(step => step.args)).toEqual([[script]]); } const all = validationPlan("mutations", { directory }); - expect(all.map(step => step.args?.[0])).toEqual(["formal/measure-semantics.mjs", "formal/measure-go-semantics.mjs"]); + expect(all.map(step => step.args?.[0])).toEqual(["formal/measure-semantics.mjs", "formal/measure-go-semantics.mjs", "formal/measure-rust-semantics.mjs"]); expect(all.some(isCompletionCheck)).toBe(false); expect(all.some(step => step.remove || step.args?.[0] === "formal/run-models.mjs")).toBe(false); }); @@ -199,6 +199,7 @@ process.exit(Number(process.argv[3] ?? 0));\n`); { label: "Measure TypeScript semantic mutations", command: process.execPath, args: ["formal/measure-semantics.mjs", "--shard=2/3"] }, ]); expect(validationPlan("mutations-go", { directory, environment: sharded }).map(step => step.args)).toEqual([["formal/measure-go-semantics.mjs", "--shard=2/3"]]); + expect(validationPlan("mutations-rust", { directory, environment: sharded }).map(step => step.args)).toEqual([["formal/measure-rust-semantics.mjs", "--shard=2/3"]]); // Unset, the plan is exactly today's complete measurement. expect(validationPlan("mutations-ts", { directory, environment }).map(step => step.args)).toEqual([["formal/measure-semantics.mjs"]]); for (const value of ["0/3", "4/3", "1/0", "a/b", "1", "01/3", "1/3/", " 1/3", ""]) { @@ -206,9 +207,9 @@ process.exit(Number(process.argv[3] ?? 0));\n`); } // The aggregates stay unsharded and refuse to ignore the variable silently; unrelated targets ignore it. for (const target of ["mutations", "ci"]) { - expect(() => validationPlan(target, { directory, environment: sharded }), target).toThrow(/MUTATION_SHARD=2\/3 applies only to make mutations-ts and make mutations-go/); + expect(() => validationPlan(target, { directory, environment: sharded }), target).toThrow(/MUTATION_SHARD=2\/3 applies only to make mutations-ts, make mutations-go and make mutations-rust/); } - for (const target of ["check", "formal", "formal-ts", "mutations-merge-ts", "mutations-merge-go"]) { + for (const target of ["check", "formal", "formal-ts", "mutations-merge-ts", "mutations-merge-go", "mutations-merge-rust"]) { expect(validationPlan(target, { directory, environment: sharded }), target).toEqual(validationPlan(target, { directory, environment })); } // The runner's environment cleaning removes replay selectors, not the shard. From 39af0a881ae3d1471b1a339e3c295427d06eb81e Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sun, 20 Sep 2026 13:59:39 -0700 Subject: [PATCH 22/31] fix(rust): close TypeScript parity gaps and strengthen conformance --- formal/measure-rust-semantics.mjs | 29 +- go/redis_interop.ts | 36 +- rust/Cargo.lock | 7 + rust/Cargo.toml | 1 + rust/README.md | 21 +- rust/src/execution.rs | 16 +- rust/src/identity.rs | 73 +--- rust/src/local.rs | 5 +- rust/src/policy.rs | 85 +++-- rust/src/shadow.rs | 14 +- rust/tests/callback_boundaries.rs | 240 +++++++++++++ rust/tests/formal/causal.rs | 211 +++++++++++ rust/tests/formal/driver.rs | 86 ++++- rust/tests/formal/fixtures.rs | 6 + rust/tests/formal/frame_vectors.rs | 96 ++--- rust/tests/formal/key_vectors.rs | 55 +-- rust/tests/formal/mod.rs | 1 + rust/tests/harness_infra.rs | 250 +++++++++++++ rust/tests/local_storage.rs | 73 ++++ rust/tests/protocol_frames.rs | 16 +- rust/tests/protocol_keys.rs | 19 +- rust/tests/redis_integration.rs | 15 +- rust/tests/redis_interop/mod.rs | 429 +++++++++++++++++++++++ rust/tests/runtime_policy.rs | 72 ++++ test/formal-rust-semantic-runner.test.ts | 43 ++- test/formal-validation.test.ts | 65 +++- 26 files changed, 1744 insertions(+), 220 deletions(-) create mode 100644 rust/tests/callback_boundaries.rs create mode 100644 rust/tests/formal/causal.rs create mode 100644 rust/tests/local_storage.rs create mode 100644 rust/tests/redis_interop/mod.rs create mode 100644 rust/tests/runtime_policy.rs diff --git a/formal/measure-rust-semantics.mjs b/formal/measure-rust-semantics.mjs index 00711738..882befbe 100644 --- a/formal/measure-rust-semantics.mjs +++ b/formal/measure-rust-semantics.mjs @@ -4,6 +4,7 @@ import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, import { resolve, relative } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; +import { causalPropertyAssertion } from './measure-go-semantics.mjs'; import { fingerprintFiles, gateDetections, languages, partitionMutations, shardDirectory, shardFromArguments } from './mutation-reports.mjs'; // The Rust counterpart of measure-go-semantics.mjs: apply each catalogued @@ -53,7 +54,8 @@ export function evaluateCargoTestOutput(output, exitCode, expectedBinaries) { // The harness writes one JSON object per line: a start header, one case per // inventory id and a finish footer. A missing footer means the run crashed or // timed out and is not evidence either way. -export function evaluateRustReport(text, exitCode) { +export function evaluateRustReport(text, exitCode, stderr = '') { + if (/^conformance harness failed:|^coverage:/m.test(stderr)) throw new Error('Rust harness infrastructure or coverage failure'); const records = text.trim().split('\n').filter(Boolean).map(line => JSON.parse(line)); if (!records.length) throw new Error('empty Rust harness report'); const [start] = records; @@ -62,19 +64,34 @@ export function evaluateRustReport(text, exitCode) { if (finish.kind !== 'finish') throw new Error('Rust harness report has no finish record: the run crashed or timed out'); const cases = records.slice(1, -1); const seen = new Set(); + const assertionKinds = {}, assertionEvidence = {}; for (const record of cases) { if (record.kind !== 'case' || typeof record.id !== 'string' || !['passed', 'failed'].includes(record.status)) throw new Error('malformed Rust harness case record'); if (seen.has(record.id)) throw new Error(`duplicate case ${record.id}`); seen.add(record.id); + const category = record.id.split('/')[0]; + if (!['sampled', 'regression', 'scenario', 'protocol', 'witness'].includes(category)) throw new Error(`unknown Rust case ${record.id}`); + if (record.status === 'failed') { + if (category === 'witness') throw new Error(`witness audit failure ${record.id}`); + const message = record.message; + if (typeof message !== 'string' || !message) throw new Error(`failure has no assertion evidence: ${record.id}`); + if (category === 'protocol') { + if (!/PROTOCOL_ASSERTION_FAILURE expected:[\s\S]*actual:/.test(message)) throw new Error(`protocol failure lacks assertion evidence: ${record.id}`); + assertionKinds[record.id] = 'protocol-assertion'; + } else if (/expected:[\s\S]*actual:/.test(message)) assertionKinds[record.id] = 'observation-mismatch'; + else if (causalPropertyAssertion(message)) assertionKinds[record.id] = 'causal-property'; + else throw new Error(`replay failure lacks observation or validated causal property evidence: ${record.id}`); + assertionEvidence[record.id] = message; + } } if (!cases.length) throw new Error('Rust harness report has no cases'); const failingTests = cases.filter(record => record.status === 'failed').map(record => record.id); - if (typeof finish.cases === 'number' && finish.cases !== cases.length) throw new Error('finish totals disagree with the case records'); - if (typeof finish.failed === 'number' && finish.failed !== failingTests.length) throw new Error('finish totals disagree with the failed cases'); + if (finish.cases !== cases.length) throw new Error('finish totals disagree with the case records'); + if (finish.failed !== failingTests.length) throw new Error('finish totals disagree with the failed cases'); if (finish.status !== (failingTests.length ? 'failed' : 'passed')) throw new Error('finish status disagrees with the case records'); if (exitCode === 0 && failingTests.length) throw new Error('harness exited 0 with failed cases'); if (exitCode !== 0 && !failingTests.length) throw new Error(`harness exited ${exitCode} without a failed case: coverage or infrastructure failure`); - return { state: failingTests.length ? 'detected' : 'survived', passed: cases.length - failingTests.length, failed: failingTests.length, failingTests, + return { state: failingTests.length ? 'detected' : 'survived', passed: cases.length - failingTests.length, failed: failingTests.length, failingTests, assertionKinds, assertionEvidence, executedTests: cases.map(record => record.id) }; } @@ -192,10 +209,10 @@ export function measureRustSemantics({ shard = { index: 1, count: 1 }, only } = writeFileSync(resolve(output, `${label}-${cohort}.log`), result.stdout ?? ''); writeFileSync(resolve(output, `${label}-${cohort}.stderr.log`), result.stderr ?? ''); if (result.error || result.signal) throw new Error(`${label}/${cohort}: runner infrastructure failed: ${result.error ?? result.signal}`); - parsed = evaluateRustReport(existsSync(reportPath) ? readFileSync(reportPath, 'utf8') : '', result.status); + parsed = evaluateRustReport(existsSync(reportPath) ? readFileSync(reportPath, 'utf8') : '', result.status, result.stderr ?? ''); } if (baseline && parsed.failed) throw new Error(`${cohort}: unmodified baseline must pass; see baseline log`); - if (!baseline && parsed.state === 'survived' && JSON.stringify([...parsed.executedTests].sort()) !== JSON.stringify([...report.baselines[cohort].executedTests].sort())) throw new Error(`${label}/${cohort}: incomplete surviving run`); + if (!baseline && JSON.stringify([...parsed.executedTests].sort()) !== JSON.stringify([...report.baselines[cohort].executedTests].sort())) throw new Error(`${label}/${cohort}: incomplete mutation run`); writeFileSync(resolve(output, `${label}-${cohort}.json`), JSON.stringify(parsed, null, 2) + '\n'); return parsed; }; diff --git a/go/redis_interop.ts b/go/redis_interop.ts index 1acdd16f..40fa8081 100644 --- a/go/redis_interop.ts +++ b/go/redis_interop.ts @@ -1,8 +1,9 @@ -// Executed by the Go integration test after bundling against production TS +// Executed by the Go and Rust integration tests after bundling production TS // imports. Inputs are operations and serialized values, never expected state. import { readFileSync } from "node:fs"; import { createClient, createCluster } from "redis"; import { createNodeRedisDialCacheClient } from "../src/node-redis.js"; +import { DialCacheKey, normalizeArgs } from "../src/key.js"; import { JsonSerializer } from "../src/serializer.js"; import { compressPayload, decompressPayload, escapeRawPayload } from "../src/internal/compression.js"; import { isRedisReadMiss } from "../src/redis-client.js"; @@ -20,26 +21,47 @@ async function main() { const results = []; try { for (const action of input.actions) { - if (action.op === "write") { + // Derive keys independently in TypeScript when the caller supplies a + // logical identity. Numeric bits avoid JSON decimal formatting becoming + // an accidental shared oracle for Number::toString regressions. + const identity = action.identity; + const args = { ...identity?.args }; + for (const [name, bits] of Object.entries(identity?.numberBits ?? {})) { + args[name] = Buffer.from(bits as string, "hex").readDoubleBE(); + } + const logical = identity === undefined ? undefined : new DialCacheKey({ + ...identity, args: normalizeArgs(args), + }); + const keys = logical === undefined ? undefined : { + logical: logical.urn, + value: `${logical.urn}:dialcache-frame-v1`, + watermark: logical.trackForInvalidation ? `${logical.prefix}#watermark` : null, + }; + const key = keys?.value ?? action.key; + const watermark = keys?.watermark ?? action.watermark; + if (action.op === "key") { + if (keys === undefined) throw new Error("key operation requires an identity"); + results.push({ kind: "key", keys }); + } else if (action.op === "write") { const serialized = action.binaryHex === undefined ? await codec.dump(action.absent ? undefined : action.value) : Buffer.from(action.binaryHex, "hex"); const payload = action.compress ? compressPayload(serialized, { thresholdBytes: 1, level: 3 }).payload : escapeRawPayload(serialized); - await adapter.write({ valueKey: action.key, value: payload, createdAtMs: action.stamp, cacheTtlMs: 60000 }); - results.push({ kind: "written" }); + await adapter.write({ valueKey: key, value: payload, createdAtMs: action.stamp, cacheTtlMs: 60000 }); + results.push({ kind: "written", ...(keys ? { keys } : {}) }); } else if (action.op === "read") { - const result = await adapter.read({ valueKey: action.key, ...(action.watermark ? { watermarkKey: action.watermark } : {}) }); + const result = await adapter.read({ valueKey: key, ...(watermark ? { watermarkKey: watermark } : {}) }); if (isRedisReadMiss(result)) { results.push(result); continue; } const { payload } = decompressPayload(result.payload); const value = action.binary ? undefined : await codec.load(payload); - results.push({ kind: "hit", stamp: result.createdAtMs, + results.push({ kind: "hit", stamp: result.createdAtMs, ...(keys ? { keys } : {}), ...(action.binary ? { binaryHex: Buffer.from(payload).toString("hex") } : { value: value === undefined ? { absent: true } : value }) }); } else if (action.op === "invalidate") { const realNow = Date.now; Date.now = () => action.stamp; - try { await adapter.invalidate({ watermarkKey: action.watermark, futureBufferMs: action.futureMs }); } + try { await adapter.invalidate({ watermarkKey: watermark, futureBufferMs: action.futureMs }); } finally { Date.now = realNow; } results.push({ kind: "invalidated" }); } else { throw new Error(`Unknown interop operation ${action.op}`); } diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 8a26a97c..6e9d6d18 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -155,6 +155,7 @@ dependencies = [ "parking_lot", "prometheus", "redis", + "ryu-js", "serde", "serde_json", "sha1", @@ -778,6 +779,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "ryu-js" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04d056b875a9d2e6cb9a61d127afee9ac5999b9f87bcb32079d1318e505be714" + [[package]] name = "scopeguard" version = "1.2.0" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index c83b3b90..9235c27e 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -24,6 +24,7 @@ parking_lot = "0.12" lru = "0.18" serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["float_roundtrip"] } +ryu-js = "1" zstd = "0.14" zstd-safe = "7" thiserror = "2" diff --git a/rust/README.md b/rust/README.md index b32b0de5..ad4774ba 100644 --- a/rust/README.md +++ b/rust/README.md @@ -85,6 +85,11 @@ invalid leaves have the narrower consequences defined in Quint (an invalid TTL or ramp disables only that layer; an invalid flag or read deadline bypasses caching for the call). +Convert a typed policy with `RuntimePolicy::from(policy)` or `policy.into()`. +Omitted leaves remain absent, including `request_local` and `coalesce`, so a +TTL-only overlay preserves the operation's flags. `Policy::to_json` uses the +same sparse representation; library defaults are applied during resolution. + Defaults are namespace `urn`, local capacity 10,000, 50 ms remote reads, 60,000 ms source calls (`SourceBudget::Default`; `SourceBudget::Unbounded` disables the deadline), sharing enabled, and shadow capacity one. Policy omits @@ -150,9 +155,12 @@ or distribution and a namespace. for `ConnectionManager`, `MultiplexedConnection` and cluster connections, routing tracked reads to slot primaries and sharing the invalidation script and frame codec with the other ports. The `redis_integration` test replays -every invalidation vector against real Redis, Valkey and Cluster servers; its -tests are `#[ignore]`d, and `make integration-rust` runs them where Docker is -available. +every invalidation vector against real Redis, Valkey and Cluster servers, +and reads/writes shared entries with the production TypeScript adapter in both +directions. Both ports construct keys independently, including numeric rounding +boundaries; the payload tests cover JSON, binary escaping, compression and +invalidation by either language. The tests are `#[ignore]`d, and +`make integration-rust` runs them where Docker is available. ## Validation and reproducing a trace @@ -203,10 +211,9 @@ every behavior-driver-backed smoke history. than a clock fault. - Prometheus collectors are shared by cloning the observer rather than by registering the same names twice. -- The behavior driver ports the effects profile's publication monitor but - not the reference drivers' cross-profile publication-causality monitor, - because `Remote::write` carries no caller context to attribute a write to - an invocation. +- The behavior driver checks source/write causality after every command. + Its test runtime carries driver-owned invocation identities across detached + tasks, independently attributing each write to its actual source callback. - The core replay, the exporters and the Redis adapter follow their Go counterparts; real-server integration is a separate lane, as in the other ports, and not part of the completion claim (see `formal/profiles.json`). diff --git a/rust/src/execution.rs b/rust/src/execution.rs index 0f526cd9..fb71f028 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -294,9 +294,12 @@ pub(crate) fn layer_labels(labels: &OutcomeLabels, layer: Layer) -> Labels { } /// Invoke the source directly with panic isolation; errors keep their identity. -async fn call_load(op: &ErasedOperation, scope: Scope) -> ValueResult { - let future = (op.load)(scope); - match AssertUnwindSafe(future).catch_unwind().await { +pub(crate) async fn call_load(op: &ErasedOperation, scope: Scope) -> ValueResult { + // Constructing a callback's future can panic before its first poll. + match AssertUnwindSafe(async { (op.load)(scope).await }) + .catch_unwind() + .await + { Ok(Ok(value)) => Ok(value), Ok(Err(error)) => Err(Error::Source(Arc::from(error))), Err(payload) => Err(Error::Panic(panic_message(payload))), @@ -840,7 +843,7 @@ impl Execution { } let start = self.elapsed(); let codec = self.op.codec.clone(); - let decoded = match AssertUnwindSafe(codec.decode(expanded.payload)) + let decoded = match AssertUnwindSafe(async { codec.decode(expanded.payload).await }) .catch_unwind() .await { @@ -885,7 +888,10 @@ impl Execution { } let start = self.elapsed(); let codec = self.op.codec.clone(); - let encoded = match AssertUnwindSafe(codec.encode(value)).catch_unwind().await { + let encoded = match AssertUnwindSafe(async { codec.encode(value).await }) + .catch_unwind() + .await + { Ok(result) => result, Err(payload) => Err(Box::new(PanicError(panic_message(payload))) as BoxError), }; diff --git a/rust/src/identity.rs b/rust/src/identity.rs index 743847ab..0afd3511 100644 --- a/rust/src/identity.rs +++ b/rust/src/identity.rs @@ -249,68 +249,10 @@ pub fn cohort(logical_key: &str, discriminator: &str) -> f64 { /// round-trips, laid out as plain decimal for exponents in `(-7, 21]` and as /// `d.ddde±X` otherwise. Negative zero spells `0`. pub fn js_number_to_string(value: f64) -> String { - if value.is_nan() { - return "NaN".to_owned(); - } - if value.is_infinite() { - return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned(); - } - if value == 0.0 { - return "0".to_owned(); - } - let (digits, exponent) = shortest_digits(value.abs()); - let k = digits.len() as i32; - let n = exponent + 1; - let mut out = String::new(); - if value < 0.0 { - out.push('-'); - } - if k <= n && n <= 21 { - out.push_str(&digits); - out.extend(std::iter::repeat_n('0', (n - k) as usize)); - } else if 0 < n && n <= 21 { - let (whole, fraction) = digits.split_at(n as usize); - out.push_str(whole); - out.push('.'); - out.push_str(fraction); - } else if -6 < n && n <= 0 { - out.push_str("0."); - out.extend(std::iter::repeat_n('0', (-n) as usize)); - out.push_str(&digits); - } else { - let (first, rest) = digits.split_at(1); - out.push_str(first); - if !rest.is_empty() { - out.push('.'); - out.push_str(rest); - } - let e = n - 1; - out.push('e'); - out.push(if e < 0 { '-' } else { '+' }); - let _ = write!(out, "{}", e.abs()); - } - out -} - -/// The shortest round-trip decimal digits of a positive finite number and -/// the decimal exponent of its first digit: `value = 0.d1d2… × 10^(exponent+1)`. -fn shortest_digits(value: f64) -> (String, i32) { - debug_assert!(value.is_finite() && value > 0.0); - // Rust's `LowerExp` without a precision prints the shortest representation - // that round-trips, as `d[.ddd]e[-]X`. - let scientific = format!("{value:e}"); - let (mantissa, exponent) = scientific - .split_once('e') - .expect("LowerExp always includes an exponent"); - let mut digits: String = mantissa.chars().filter(|c| *c != '.').collect(); - let trimmed = digits.trim_end_matches('0').len().max(1); - digits.truncate(trimmed); - ( - digits, - exponent - .parse() - .expect("LowerExp exponent is a decimal integer"), - ) + // Rust's standard shortest formatter rounds some decimal ties away from + // zero; ECMAScript chooses the even significand. Those spellings are part + // of the shared key identity, so use the ECMAScript variant of Ryū. + ryu_js::Buffer::new().format(value).to_owned() } /// Compare two strings by UTF-16 code units, as JavaScript's `<` does. @@ -348,6 +290,13 @@ mod tests { (0.5, "0.5"), (123.456, "123.456"), (1e20, "100000000000000000000"), + // Exact binary fractions whose shortest decimal candidates tie. + (f64::from_bits(0x430c6bf526340002), "1000000000000000.2"), + (f64::from_bits(0xc30c6bf526340002), "-1000000000000000.2"), + (f64::from_bits(0x430c6bf526340006), "1000000000000000.8"), + (f64::from_bits(0xc30c6bf526340006), "-1000000000000000.8"), + (f64::from_bits(0x42d6bcc41e900008), "100000000000000.12"), + (f64::from_bits(0x42d6bcc41e900018), "100000000000000.38"), ]; for (value, expected) in cases { assert_eq!( diff --git a/rust/src/local.rs b/rust/src/local.rs index 1c086fdc..494d47ef 100644 --- a/rust/src/local.rs +++ b/rust/src/local.rs @@ -59,10 +59,11 @@ pub struct LruLocalStore { } impl LruLocalStore { - /// A store holding at most `capacity` entries. Capacity must be positive. + /// A store holding at most `capacity` entries. Capacity must be positive; + /// storage is allocated as entries arrive rather than for the full limit. pub fn new(capacity: NonZeroUsize) -> Self { LruLocalStore { - entries: lru::LruCache::new(capacity), + entries: lru::LruCache::sparse(capacity), } } diff --git a/rust/src/policy.rs b/rust/src/policy.rs index 495e23e6..78c890a7 100644 --- a/rust/src/policy.rs +++ b/rust/src/policy.rs @@ -207,9 +207,9 @@ impl Policy { /// The JSON-shaped form of this policy, with omitted leaves absent. /// - /// `ttlSec` and `ramp` are always objects (possibly empty); `requestLocal` - /// (default false) and `coalesce` (default true) are always present; - /// the optional scalars and the `shadow` object appear only when set. + /// `ttlSec` and `ramp` are always objects (possibly empty). All other + /// leaves, including `requestLocal` and `coalesce`, appear only when set, + /// so converting this policy to a runtime overlay preserves inheritance. pub fn to_json(&self) -> Value { let mut ttl = Map::new(); if let Some(seconds) = self.local_ttl_sec { @@ -228,14 +228,12 @@ impl Policy { let mut object = Map::new(); object.insert("ttlSec".to_string(), Value::Object(ttl)); object.insert("ramp".to_string(), Value::Object(ramp)); - object.insert( - "requestLocal".to_string(), - Value::Bool(self.request_local.unwrap_or(false)), - ); - object.insert( - "coalesce".to_string(), - Value::Bool(self.coalesce.unwrap_or(true)), - ); + if let Some(enabled) = self.request_local { + object.insert("requestLocal".to_string(), Value::Bool(enabled)); + } + if let Some(enabled) = self.coalesce { + object.insert("coalesce".to_string(), Value::Bool(enabled)); + } if let Some(seconds) = self.stale_on_error_max_age_sec { object.insert("staleOnErrorMaxAgeSec".to_string(), Value::from(seconds)); } @@ -445,6 +443,10 @@ pub fn resolve_policy( Value::Object(object) => object, _ => unreachable!("Policy::to_json always produces an object"), }; + // Defaults belong to resolution, not the sparse policy representation. + // Present overlay values (including invalid nulls) still replace them. + merged.entry("requestLocal").or_insert(Value::Bool(false)); + merged.entry("coalesce").or_insert(Value::Bool(true)); if let Some(RuntimePolicy(overlay)) = overlay { if !overlay.is_null() { merge_overlay(&mut merged, overlay)?; @@ -934,10 +936,10 @@ mod tests { #[test] fn to_json_matches_the_static_policy_map_shape() { - // Empty policy: layer maps present but empty, flags at their defaults, optionals absent. + // Empty policy: layer maps present but empty, all other leaves absent. assert_eq!( Policy::default().to_json(), - json!({"ttlSec": {}, "ramp": {}, "requestLocal": false, "coalesce": true}) + json!({"ttlSec": {}, "ramp": {}}) ); // Only present leaves appear inside the layer maps; integral ramps spell without a fraction. assert_eq!( @@ -945,11 +947,11 @@ mod tests { .local_ttl_sec(7) .remote_ramp(40.0) .to_json(), - json!({"ttlSec": {"local": 7}, "ramp": {"remote": 40}, "requestLocal": false, "coalesce": true}) + json!({"ttlSec": {"local": 7}, "ramp": {"remote": 40}}) ); assert_eq!( Policy::default().local_ramp(12.5).to_json(), - json!({"ttlSec": {}, "ramp": {"local": 12.5}, "requestLocal": false, "coalesce": true}) + json!({"ttlSec": {}, "ramp": {"local": 12.5}}) ); // An explicit flag value is emitted even when it equals the default. assert_eq!( @@ -962,7 +964,7 @@ mod tests { // A present but empty shadow policy is an empty object. assert_eq!( Policy::default().shadow(ShadowPolicy::default()).to_json(), - json!({"ttlSec": {}, "ramp": {}, "requestLocal": false, "coalesce": true, "shadow": {}}) + json!({"ttlSec": {}, "ramp": {}, "shadow": {}}) ); assert_eq!( Policy::default() @@ -972,7 +974,7 @@ mod tests { }) .to_json(), json!({ - "ttlSec": {}, "ramp": {}, "requestLocal": false, "coalesce": true, + "ttlSec": {}, "ramp": {}, "shadow": {"logMismatches": false}, }) ); @@ -983,7 +985,7 @@ mod tests { .remote_read_timeout_ms(75) .to_json(), json!({ - "ttlSec": {}, "ramp": {}, "requestLocal": false, "coalesce": true, + "ttlSec": {}, "ramp": {}, "staleOnErrorMaxAgeSec": 0, "remoteReadTimeoutMs": 75, }) ); @@ -992,7 +994,6 @@ mod tests { json!({ "ttlSec": {"local": 60, "remote": 60}, "ramp": {"local": 100, "remote": 100}, - "requestLocal": false, "coalesce": true, }) ); assert_eq!( @@ -1000,7 +1001,7 @@ mod tests { json!({ "ttlSec": {}, "ramp": {"local": 0, "remote": 0}, - "requestLocal": false, "coalesce": true, + "requestLocal": false, "staleOnErrorMaxAgeSec": 0, "shadow": {"ramp": 0, "logMismatches": false}, }) @@ -1445,6 +1446,50 @@ mod tests { assert_eq!(r.remote.reason, Some(DisabledReason::PolicyDisabled)); } + #[test] + fn typed_runtime_overlays_preserve_omitted_flags() { + let base = Policy::default().request_local(true).coalesce(false); + for overlay in [Policy::default(), Policy::default().local_ttl_sec(9)] { + let r = resolve_policy( + &base, + Some(&RuntimePolicy::from(overlay)), + KEY, + PolicyDefaults::default(), + ) + .unwrap(); + assert!(r.request_local, "an omitted flag must inherit the baseline"); + assert!(!r.coalesce, "an omitted flag must inherit the baseline"); + } + + let explicit = resolved( + &base, + Some(RuntimePolicy::from(Policy::default().request_local(false).coalesce(true)).0), + ); + assert!(!explicit.request_local); + assert!(explicit.coalesce); + + let disabled = resolved(&base, Some(RuntimePolicy::from(Policy::disabled()).0)); + assert!(!disabled.request_local); + assert!( + !disabled.coalesce, + "the kill switch leaves coalescing unset" + ); + } + + #[test] + fn json_round_trip_preserves_omitted_and_explicit_flags() { + for request_local in [None, Some(false), Some(true)] { + for coalesce in [None, Some(false), Some(true)] { + let policy = Policy { + request_local, + coalesce, + ..Policy::default() + }; + assert_eq!(Policy::from_json(&policy.to_json()).unwrap(), policy); + } + } + } + #[test] fn instance_defaults_and_deadline_precedence() { let base = Policy::enabled(1); diff --git a/rust/src/shadow.rs b/rust/src/shadow.rs index c65cdd6d..a5036b59 100644 --- a/rust/src/shadow.rs +++ b/rust/src/shadow.rs @@ -13,8 +13,8 @@ use parking_lot::Mutex; use crate::deadline::{await_deadline, since}; use crate::engine::Core; use crate::error::Error; -use crate::execution::{Execution, RawRead}; -use crate::flight::{panic_message, start_pending, yield_deferred, Settled, ValueResult}; +use crate::execution::{call_load, Execution, RawRead}; +use crate::flight::{start_pending, yield_deferred, Settled, ValueResult}; use crate::limits::DEFAULT_FALLBACK_TIMEOUT_MS; use crate::local::StoredValue; use crate::observe::{ @@ -371,15 +371,7 @@ impl Execution { } result } - None => { - let disabled = self.scope.disabled_view(); - let future = (self.op.load)(disabled); - match AssertUnwindSafe(future).catch_unwind().await { - Ok(Ok(value)) => Ok(value), - Ok(Err(error)) => Err(Error::Source(Arc::from(error))), - Err(payload) => Err(Error::Panic(panic_message(payload))), - } - } + None => call_load(&self.op, self.scope.disabled_view()).await, }; let value = match value { Ok(value) => value, diff --git a/rust/tests/callback_boundaries.rs b/rust/tests/callback_boundaries.rs new file mode 100644 index 00000000..c1370327 --- /dev/null +++ b/rust/tests/callback_boundaries.rs @@ -0,0 +1,240 @@ +//! Native callback boundaries: a callback may panic while constructing its +//! future, before the asynchronous effects exercised by the portable replay. + +use std::future::{ready, Ready}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; + +use dialcache::observe::{ErrorKind, RecoveryOutcome, ShadowOutcome}; +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{ + BoxError, DialCache, Event, Frame, FromSync, Identity, InvalidateRequest, JsonCodec, + MissReason, Observer, Operation, Payload, Policy, ReadContext, ReadRequest, ReadResult, Remote, + ShadowPolicy, SyncCodec, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; + +struct SnapshotRemote { + result: ReadResult, + writes: AtomicUsize, +} + +impl Remote for SnapshotRemote { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(ready(Ok(self.result.clone()))) + } + + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + self.writes.fetch_add(1, Ordering::SeqCst); + Box::pin(ready(Ok(()))) + } + + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(ready(Ok(()))) + } +} + +#[derive(Default)] +struct Events(Mutex>); + +impl Observer for Events { + fn observe(&self, event: &Event) { + self.0.lock().push(event.clone()); + } + + fn observes_shadow_outcomes(&self) -> bool { + true + } +} + +fn setup(result: ReadResult) -> (TestExecutor, DialCache, Arc, Arc) { + let executor = TestExecutor::new(WALL_EPOCH_MS); + let remote = Arc::new(SnapshotRemote { + result, + writes: AtomicUsize::new(0), + }); + let events = Arc::new(Events::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote_arc(remote.clone()) + .observer_arc(events.clone()) + .build() + .expect("valid configuration"); + (executor, cache, remote, events) +} + +fn hit(age_ms: u64) -> ReadResult { + ReadResult::Hit(Frame { + created_at_ms: WALL_EPOCH_MS as u64 - age_ms, + payload: Payload::text("1"), + }) +} + +enum PanickingCodec { + Encode, + Decode, +} + +impl SyncCodec for PanickingCodec { + fn encode(&self, value: &u64) -> Result { + if matches!(self, Self::Encode) { + panic!("synchronous encode failure"); + } + JsonCodec::encode_value(value) + } + + fn decode(&self, payload: Payload) -> Result { + if matches!(self, Self::Decode) { + panic!("synchronous decode failure"); + } + JsonCodec::decode_value(&payload) + } +} + +fn operation(codec: PanickingCodec) -> Operation { + Operation::with_codec( + Identity::new("thing", "one", "CallbackBoundary"), + Arc::new(FromSync(codec)), + |a, b| a == b, + ) + .policy(Policy::default().remote_ttl_sec(60)) +} + +#[test] +fn synchronous_encode_panic_preserves_source_success_and_local_publication() { + let (mut executor, cache, remote, events) = setup(ReadResult::miss(MissReason::ValueAbsent)); + let result = executor.block_on(async move { + let request = cache.enable_guard(); + let operation = operation(PanickingCodec::Encode).policy(Policy::enabled(60)); + let first = cache + .get_or_load(request.scope(), operation.clone(), |_| ready(Ok(7))) + .await?; + let second = cache + .get_or_load( + request.scope(), + operation, + |_| -> Ready> { + panic!("the source must not run after local publication") + }, + ) + .await?; + Ok::<_, dialcache::Error>((first, second)) + }); + let (first, second) = result.expect("a codec panic must fail open"); + assert_eq!((*first, *second), (7, 7)); + assert_eq!(remote.writes.load(Ordering::SeqCst), 0); + assert!(events.0.lock().iter().any(|event| matches!( + event, + Event::Error { + error: ErrorKind::SerializationDump, + .. + } + ))); +} + +#[test] +fn synchronous_decode_panic_falls_through_to_source_and_refills() { + let (mut executor, cache, remote, events) = setup(hit(0)); + let result = executor.block_on(async move { + let request = cache.enable_guard(); + cache + .get_or_load(request.scope(), operation(PanickingCodec::Decode), |_| { + ready(Ok(7)) + }) + .await + }); + assert_eq!(*result.expect("decode panic must reach the source"), 7); + assert_eq!(remote.writes.load(Ordering::SeqCst), 1); + assert!(events.0.lock().iter().any(|event| matches!( + event, + Event::Error { + error: ErrorKind::SerializationLoad, + .. + } + ))); +} + +#[test] +fn synchronous_recovery_decode_panic_preserves_the_source_error() { + let (mut executor, cache, remote, events) = setup(hit(2_000)); + let result = executor.block_on(async move { + let request = cache.enable_guard(); + let operation = operation(PanickingCodec::Decode) + .policy( + Policy::default() + .remote_ttl_sec(1) + .stale_on_error_max_age_sec(3), + ) + .should_recover(|_| true); + cache + .get_or_load(request.scope(), operation, |_| { + ready(Err("source failed".into())) + }) + .await + }); + let error = result.expect_err("recovery cannot decode the retained frame"); + assert_eq!( + error + .source_error() + .expect("original source error") + .to_string(), + "source failed" + ); + assert_eq!(remote.writes.load(Ordering::SeqCst), 0); + assert!(events.0.lock().iter().any(|event| matches!( + event, + Event::StaleRecovery { + outcome: RecoveryOutcome::DeserializationError, + .. + } + ))); +} + +#[test] +fn synchronous_shadow_source_panic_is_a_source_error_and_releases_capacity() { + let (mut executor, cache, _, events) = setup(hit(0)); + let request = cache.enable_guard(); + let source_enabled = Arc::new(AtomicBool::new(true)); + for _ in 0..2 { + let cache = cache.clone(); + let scope = request.scope().clone(); + let source_enabled = source_enabled.clone(); + let result = executor.block_on(async move { + let operation = Operation::::new(Identity::new("thing", "one", "ShadowBoundary")) + .policy(Policy::default().remote_ttl_sec(60).shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: None, + })); + cache + .get_or_load( + &scope, + operation, + move |scope| -> Ready> { + source_enabled.store(scope.is_enabled(), Ordering::SeqCst); + panic!("synchronous source failure"); + }, + ) + .await + }); + assert_eq!(*result.expect("served hit survives shadow failure"), 1); + } + assert!( + !source_enabled.load(Ordering::SeqCst), + "shadow source must run disabled" + ); + let outcomes: Vec<_> = events + .0 + .lock() + .iter() + .filter_map(|event| match event { + Event::ShadowValidation { outcome, .. } => Some(*outcome), + _ => None, + }) + .collect(); + assert_eq!( + outcomes, + vec![ShadowOutcome::SourceError, ShadowOutcome::SourceError] + ); +} diff --git a/rust/tests/formal/causal.rs b/rust/tests/formal/causal.rs new file mode 100644 index 00000000..7308fe25 --- /dev/null +++ b/rust/tests/formal/causal.rs @@ -0,0 +1,211 @@ +//! Independent publication checks over driver-owned invocation contexts and +//! actual source/write callbacks. No model predictions or cache internals enter +//! this journal. The runtime decorator preserves context across detached tasks. + +use std::cell::Cell; +use std::collections::{HashMap, HashSet}; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use std::time::Duration; + +use dialcache::Runtime; +use futures::future::BoxFuture; +use serde_json::{json, Value}; + +thread_local! { + static INVOCATION: Cell> = const { Cell::new(None) }; +} + +pub fn current_invocation() -> Option { + INVOCATION.get() +} + +struct RestoreInvocation(Option); + +impl Drop for RestoreInvocation { + fn drop(&mut self) { + INVOCATION.set(self.0); + } +} + +/// Installs the owner only while polling, so unrelated interleaved tasks never +/// inherit it. Restoration also happens if the wrapped future panics. +pub struct InvocationFuture { + owner: Option, + future: Pin>, +} + +impl InvocationFuture { + pub fn new(owner: Option, future: F) -> Self { + Self { + owner, + future: Box::pin(future), + } + } +} + +impl Future for InvocationFuture { + type Output = F::Output; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + let _restore = RestoreInvocation(INVOCATION.replace(this.owner)); + this.future.as_mut().poll(cx) + } +} + +pub struct InvocationRuntime(pub Arc); + +impl Runtime for InvocationRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.0 + .spawn(Box::pin(InvocationFuture::new(current_invocation(), task))); + } + + fn defer(&self, task: BoxFuture<'static, ()>) { + self.0 + .defer(Box::pin(InvocationFuture::new(current_invocation(), task))); + } + + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { + self.0.sleep(duration) + } +} + +#[derive(Clone, Debug)] +pub enum CausalEvent { + SourceStart { + id: usize, + owner: usize, + at_ms: i64, + budget_ms: Option, + }, + SourceSettlement { + id: usize, + at_ms: i64, + outcome: &'static str, + }, + WriteDispatch { + source: Option, + owner: Option, + at_ms: i64, + }, +} + +/// Only semantically valid journals can reach this discriminator. The mutation +/// measurement independently validates its rule and evidence fields. +pub fn property_failure(rule: &str, condition: &str, mut event: Value) -> String { + event["condition"] = Value::from(condition); + format!("CAUSAL_PROPERTY_FAILURE rule={rule} event={event}") +} + +/// Necessary C25/C26 conditions; fence and payload provenance still have their +/// separate replay assertions. Pending prefixes and late write completion are +/// allowed when the exact source settled successfully within its own budget. +pub fn assert_publication_causality(history: &[CausalEvent]) -> Result<(), String> { + struct Source { + owner: usize, + started: i64, + budget: Option, + settled: Option, + outcome: &'static str, + } + let mut sources: HashMap = HashMap::new(); + let mut owners = HashSet::new(); + let mut previous = -1; + for (index, event) in history.iter().enumerate() { + let fail = |reason| format!("C25/C26 causal event {index}: {reason}"); + let at_ms = match *event { + CausalEvent::SourceStart { at_ms, .. } + | CausalEvent::SourceSettlement { at_ms, .. } + | CausalEvent::WriteDispatch { at_ms, .. } => at_ms, + }; + if at_ms < 0 || at_ms < previous { + return Err(fail( + "elapsed observations must be nonnegative and monotonic", + )); + } + previous = at_ms; + match *event { + CausalEvent::SourceStart { + id, + owner, + at_ms, + budget_ms, + } => { + if sources.contains_key(&id) || !owners.insert(owner) { + return Err(fail("source and invocation ownership must be unique")); + } + if budget_ms == Some(0) { + return Err(fail( + "source budget must be positive or explicitly unbounded", + )); + } + sources.insert( + id, + Source { + owner, + started: at_ms, + budget: budget_ms, + settled: None, + outcome: "", + }, + ); + } + CausalEvent::SourceSettlement { id, at_ms, outcome } => { + let source = sources + .get_mut(&id) + .ok_or_else(|| fail("settlement must identify one actual pending source"))?; + if source.settled.is_some() || !["resolve", "reject"].contains(&outcome) { + return Err(fail("settlement must identify one actual pending source")); + } + source.settled = Some(at_ms); + source.outcome = outcome; + } + CausalEvent::WriteDispatch { source, owner, .. } => { + let (Some(id), Some(owner)) = (source, owner) else { + return Err(fail("write has no observed source ownership")); + }; + let source = sources + .get(&id) + .ok_or_else(|| fail("write has no observed source ownership"))?; + let evidence = json!({"event":"writeDispatch", "index":index, "atMs":at_ms, "source":id, "owner":owner}); + if source.owner != owner { + let mut evidence = evidence; + evidence["sourceOwner"] = json!(source.owner); + return Err(property_failure( + "C26", + "write belongs to a different invocation's source", + evidence, + )); + } + if source.outcome != "resolve" { + let mut evidence = evidence; + evidence["outcome"] = json!(source.outcome); + return Err(property_failure( + "C26", + "write requires that exact source's successful settlement", + evidence, + )); + } + if let Some(budget) = source.budget { + let settled = source.settled.expect("resolved source has a settlement"); + if (settled - source.started) as u64 >= budget { + let mut evidence = evidence; + evidence["startedAtMs"] = json!(source.started); + evidence["settledAtMs"] = json!(settled); + evidence["budgetMs"] = json!(budget); + return Err(property_failure( + "C25", + "late raw settlement cannot authorize publication", + evidence, + )); + } + } + } + } + } + Ok(()) +} diff --git a/rust/tests/formal/driver.rs b/rust/tests/formal/driver.rs index e2b5c02e..7fc4b5c7 100644 --- a/rust/tests/formal/driver.rs +++ b/rust/tests/formal/driver.rs @@ -19,6 +19,10 @@ use futures::future::BoxFuture; use parking_lot::Mutex; use serde_json::{json, Map, Value}; +use super::causal::{ + assert_publication_causality, current_invocation, property_failure, CausalEvent, + InvocationFuture, InvocationRuntime, +}; use super::gate::Gate; use super::json::json_equal; @@ -126,6 +130,8 @@ pub struct Shared { reply: Option, effects: HashMap>>>, history: Vec, + causal_history: Vec, + source_by_invocation: HashMap, fallback_failed: bool, pub discard_writes: bool, pub discard_invalidations: bool, @@ -318,6 +324,8 @@ impl Driver { reply: None, effects: HashMap::new(), history: Vec::new(), + causal_history: Vec::new(), + source_by_invocation: HashMap::new(), fallback_failed: false, discard_writes: false, discard_invalidations: false, @@ -387,7 +395,7 @@ impl Driver { .unwrap_or(10_000); let mut builder = DialCache::builder() .clock_arc(self.clock.clone()) - .runtime_arc(self.exec.runtime.clone()) + .runtime(InvocationRuntime(self.exec.runtime.clone())) .disable_compression() .local_capacity(capacity) .shadow_max_in_flight( @@ -513,6 +521,11 @@ impl Driver { gate.settle(LoaderOutcome::Value(value)); "resolve" }; + s.causal_history.push(CausalEvent::SourceSettlement { + id: index, + at_ms: at, + outcome, + }); s.history.push(HistoryEvent { event: "sourceSettlement", id: index, @@ -675,7 +688,7 @@ impl Driver { // Drain ready work while unresolved external gates remain held: the // causally-ready-v1 settlement step. self.exec.drain(); - Ok(()) + self.assert_publication_causality() } fn begin(&mut self, input: &Value) -> Result<(), String> { @@ -751,10 +764,14 @@ impl Driver { let source_cache = cache.clone(); let source_shared = shared.clone(); let source_clock = clock.clone(); + // Filled from the actual scope when this driver-owned call starts. + let source_budget = Arc::new(Mutex::new(None)); + let loader_budget = source_budget.clone(); let load = move |scope: Scope| { let shared = source_shared.clone(); let clock = source_clock.clone(); let cache = source_cache.clone(); + let source_budget = loader_budget.clone(); async move { let gate = { let mut s = shared.lock(); @@ -764,6 +781,13 @@ impl Driver { s.increment("loaders"); use dialcache::Clock; let at = clock.elapsed_ms(); + s.source_by_invocation.insert(index, id); + s.causal_history.push(CausalEvent::SourceStart { + id, + owner: index, + at_ms: at, + budget_ms: *source_budget.lock(), + }); s.history.push(HistoryEvent { event: "sourceStart", id, @@ -799,7 +823,17 @@ impl Driver { let cache = call_cache.clone(); let operation = operation.clone(); let load = load.clone(); - async move { + let source_budget = source_budget.clone(); + InvocationFuture::new(Some(index), async move { + *source_budget.lock() = if !cache.is_enabled(&scope) { + None + } else { + match operation.budget { + SourceBudget::Default => Some(60_000), + SourceBudget::Unbounded => None, + SourceBudget::Millis(ms) => Some(ms), + } + }; let result = cache.get_or_load(&scope, operation, load).await; let mut s = shared.lock(); let record = match result { @@ -809,7 +843,7 @@ impl Driver { if let Some(Value::Array(calls)) = s.observed.get_mut("calls") { calls[index] = record; } - } + }) }; let disabled = input.get("disabled") == Some(&Value::Bool(true)); let outside = input.get("outside") == Some(&Value::Bool(true)); @@ -951,17 +985,28 @@ impl Driver { } } + pub fn causal_history(&self) -> Vec { + self.shared.lock().causal_history.clone() + } + + pub fn assert_publication_causality(&self) -> Result<(), String> { + assert_publication_causality(&self.causal_history()) + } + pub fn history(&self) -> Vec { self.shared.lock().history.clone() } /// C23/C25/C26 monitor over the actual callback history (effects profile). pub fn assert_effects_history(&self) -> Result<(), String> { + Self::assert_effects_events(&self.history()) + } + + pub fn assert_effects_events(history: &[HistoryEvent]) -> Result<(), String> { struct Source { at: i64, settled: &'static str, } - let history = self.history(); let mut sources: HashMap = HashMap::new(); let mut active: Option = None; let mut authorized = false; @@ -1013,21 +1058,35 @@ impl Driver { } let elapsed = (e.at - source.at) as f64; if (e.duration_ms - elapsed).abs() > 1e-7 { - return Err(format!("C23: duration includes lookup or omits source time (event {index}, at {} elapsed {elapsed} duration {})", e.at, e.duration_ms)); + return Err(property_failure( + "C23", + "duration includes lookup or omits source time", + json!({"event":e.event, "index":index, "atMs":e.at, "elapsedMs":elapsed, "durationMs":e.duration_ms}), + )); } if !e.failed && (elapsed >= 10.0 || source.settled != "resolve") { - return Err(format!("C25: success must be accepted before its source deadline (event {index}, elapsed {elapsed}, settlement {})", source.settled)); + return Err(property_failure( + "C25", + "success must be accepted before its source deadline", + json!({"event":e.event, "index":index, "atMs":e.at, "elapsedMs":elapsed, "budgetMs":10, "settlement":source.settled, "failed":e.failed}), + )); } if e.failed && elapsed < 10.0 && source.settled != "reject" { - return Err(format!("C23: source lost its full source-relative budget (event {index}, elapsed {elapsed}, settlement {})", source.settled)); + return Err(property_failure( + "C23", + "source lost its full source-relative budget", + json!({"event":e.event, "index":index, "atMs":e.at, "elapsedMs":elapsed, "budgetMs":10, "settlement":source.settled, "failed":e.failed}), + )); } active = None; authorized = !e.failed; } "writeDispatch" => { if !authorized { - return Err(format!( - "C26: publication without accepted source success (event {index})" + return Err(property_failure( + "C26", + "publication without accepted source success", + json!({"event":e.event, "index":index, "atMs":e.at, "authorized":false}), )); } } @@ -1176,6 +1235,13 @@ impl Remote for DriverRemote { let mut s = shared.lock(); let index = s.increment("writes"); let at = clock.elapsed_ms(); + let owner = current_invocation(); + let source = owner.and_then(|owner| s.source_by_invocation.get(&owner).copied()); + s.causal_history.push(CausalEvent::WriteDispatch { + source, + owner, + at_ms: at, + }); s.history.push(HistoryEvent { event: "writeDispatch", id: 0, diff --git a/rust/tests/formal/fixtures.rs b/rust/tests/formal/fixtures.rs index 7e4f2c26..de42b8e5 100644 --- a/rust/tests/formal/fixtures.rs +++ b/rust/tests/formal/fixtures.rs @@ -341,6 +341,12 @@ pub fn protocol_groups(selection: &str) -> BTreeMap> { groups } +/// An executed protocol assertion, distinguished from malformed fixture input +/// and environmental codec failures by the mutation measurement. +pub fn assertion_mismatch(expected: impl std::fmt::Debug, actual: impl std::fmt::Debug) -> String { + format!("PROTOCOL_ASSERTION_FAILURE expected: {expected:?}\nactual: {actual:?}") +} + #[cfg(test)] mod tests { #[allow(unused_imports)] diff --git a/rust/tests/formal/frame_vectors.rs b/rust/tests/formal/frame_vectors.rs index a07baa17..405499ca 100644 --- a/rust/tests/formal/frame_vectors.rs +++ b/rust/tests/formal/frame_vectors.rs @@ -12,6 +12,8 @@ use dialcache::protocol::{ use dialcache::{Frame, Payload, ReadResult}; use serde_json::{json, Value}; +use super::fixtures::assertion_mismatch; + use super::fixtures::LONE_SURROGATE_MARKER; /// The fixture level every `codecBytes` entry was measured at. @@ -90,15 +92,15 @@ pub fn check_frame_vector(vector: &Value) -> Result<(), String> { .as_u64() .ok_or("createdAtMs is not an unsigned integer")?; let payload = vector_payload(vector, true)?; + let expected = text_field(vector, "frameHex")?; let frame = encode_frame(&Frame { created_at_ms, payload, }) - .map_err(|error| format!("encode failed: {error}"))?; + .map_err(|error| assertion_mismatch(expected, error))?; let actual = hex::encode(&frame); - let expected = text_field(vector, "frameHex")?; if actual != expected { - return Err(format!("frame: got {actual}, want {expected}")); + return Err(assertion_mismatch(expected, actual)); } Ok(()) } @@ -126,9 +128,10 @@ fn check_decode_vector(vector: &Value, tracked: bool) -> Result<(), String> { Value::String(text) => Some(text.as_bytes().to_vec()), _ => return Err("watermarkUtf8 is neither null nor a string".to_string()), }; + let expected = field(vector, "expected")?; let actual = match decode_frame(raw.as_deref(), tracked, watermark.as_deref()) { Err(ProtocolError::PayloadEncoding) => json!({ "kind": "payload_encoding_error" }), - Err(other) => return Err(format!("unexpected decode error {other}")), + Err(other) => return Err(assertion_mismatch(expected, other)), Ok(ReadResult::Miss { reason, observed_watermark_ms, @@ -147,15 +150,14 @@ fn check_decode_vector(vector: &Value, tracked: bool) -> Result<(), String> { } else { object["payloadType"] = json!("string"); let text = String::from_utf8(frame.payload.bytes) - .map_err(|_| "text payload is not UTF-8")?; + .map_err(|error| assertion_mismatch(expected, error))?; object["payloadUtf8"] = json!(text); } object } }; - let expected = field(vector, "expected")?; if &actual != expected { - return Err(format!("got {actual}\nwant {expected}")); + return Err(assertion_mismatch(expected, actual)); } Ok(()) } @@ -163,19 +165,27 @@ fn check_decode_vector(vector: &Value, tracked: bool) -> Result<(), String> { /// `invalidTimestampVectors`: every input is outside the timestamp domain. pub fn check_invalid_timestamp_vector(vector: &Value) -> Result<(), String> { match validate_timestamp_ms(number_input(vector)?) { - Ok(accepted) => Err(format!("invalid timestamp accepted as {accepted}")), + Ok(accepted) => Err(assertion_mismatch("invalid timestamp rejected", accepted)), Err(_) => Ok(()), } } /// `durationVectors`: `expected` null rejects; otherwise the ceiled TTL. pub fn check_duration_vector(vector: &Value) -> Result<(), String> { + let expected = match field(vector, "expected")? { + Value::Null => None, + value => Some( + value + .as_u64() + .ok_or("expected is neither null nor an unsigned integer")?, + ), + }; let result = ceil_supported_cache_ttl_ms(number_input(vector)?); - match (field(vector, "expected")?.as_u64(), result) { + match (expected, result) { (None, Err(_)) => Ok(()), - (None, Ok(accepted)) => Err(format!("invalid duration accepted as {accepted}")), + (None, Ok(accepted)) => Err(assertion_mismatch("invalid duration rejected", accepted)), (Some(expected), Ok(actual)) if actual == expected => Ok(()), - (Some(expected), other) => Err(format!("got {other:?} want {expected}")), + (Some(expected), other) => Err(assertion_mismatch(expected, other)), } } @@ -185,7 +195,10 @@ pub fn check_envelope_vector(vector: &Value) -> Result<(), String> { let raw = Payload::binary(unhex(text_field(vector, "inputHex")?)?); let escaped = escape_raw_payload(raw.clone()); if escaped.bytes != unhex(text_field(vector, "escapedHex")?)? { - return Err(format!("escape differs: {}", hex::encode(&escaped.bytes))); + return Err(assertion_mismatch( + text_field(vector, "escapedHex")?, + hex::encode(&escaped.bytes), + )); } let decoded = decompress_payload(raw.clone(), MAX_DECOMPRESSED_BYTES); let expected_outcome = text_field(vector, "outcome")?; @@ -193,9 +206,9 @@ pub fn check_envelope_vector(vector: &Value) -> Result<(), String> { || !decoded.payload.binary || decoded.payload.bytes != unhex(text_field(vector, "decodedHex")?)? { - return Err(format!( - "decoded {decoded:?}, want {expected_outcome} {}", - text_field(vector, "decodedHex")? + return Err(assertion_mismatch( + (expected_outcome, text_field(vector, "decodedHex")?), + &decoded, )); } if decompress_payload(escaped, MAX_DECOMPRESSED_BYTES) @@ -203,7 +216,7 @@ pub fn check_envelope_vector(vector: &Value) -> Result<(), String> { .bytes != raw.bytes { - return Err("escape roundtrip differs".to_string()); + return Err(assertion_mismatch(&raw.bytes, "escape roundtrip differs")); } Ok(()) } @@ -233,14 +246,15 @@ pub fn check_compressed_decode_vector(vector: &Value) -> Result<(), String> { (false, Err(_)) => {} } } - let got = decompress_payload(Payload::binary(input), max_decompressed_bytes(vector)?); let want = vector_payload(vector, false)?; + let max = max_decompressed_bytes(vector)?; let outcome = vector .get("outcome") .and_then(Value::as_str) .unwrap_or("decompressed"); + let got = decompress_payload(Payload::binary(input), max); if outcome_name(got.outcome) != outcome || got.payload != want { - return Err(format!("got {got:?} want {outcome} {want:?}")); + return Err(assertion_mismatch((outcome, want), got)); } Ok(()) } @@ -271,6 +285,10 @@ pub fn check_compression_write_vector(vector: &Value) -> Result<(), String> { } } + let expected_outcome = match modeled { + Some(expected) => text_field(expected, "outcome")?, + None => text_field(vector, "outcome")?, + }; let got = compress_payload( raw.clone(), &CompressionConfig { @@ -279,21 +297,14 @@ pub fn check_compression_write_vector(vector: &Value) -> Result<(), String> { }, max, ) - .map_err(|error| format!("compress failed: {error}"))?; - let expected_outcome = match modeled { - Some(expected) => text_field(expected, "outcome")?, - None => text_field(vector, "outcome")?, - }; + .map_err(|error| assertion_mismatch(expected_outcome, error))?; if got.outcome.as_str() != expected_outcome { - return Err(format!( - "got outcome {} want {expected_outcome} ({got:?})", - got.outcome - )); + return Err(assertion_mismatch(expected_outcome, &got)); } if let Some(expected) = modeled { let stored = usize_field(expected, "storedBytes")?; if got.stored_bytes != stored { - return Err(format!("storedBytes {} want {stored}", got.stored_bytes)); + return Err(assertion_mismatch(stored, got.stored_bytes)); } if vector.get("originalBytes").is_none() || vector.get("rawStoredBytes").is_none() @@ -307,10 +318,7 @@ pub fn check_compression_write_vector(vector: &Value) -> Result<(), String> { if let Some(original) = vector.get("originalBytes").filter(|value| !value.is_null()) { let original = original.as_u64().ok_or("originalBytes is not a number")? as usize; if got.original_bytes != original { - return Err(format!( - "originalBytes {} want {original}", - got.original_bytes - )); + return Err(assertion_mismatch(original, got.original_bytes)); } } if let Some(raw_stored) = vector @@ -321,31 +329,28 @@ pub fn check_compression_write_vector(vector: &Value) -> Result<(), String> { .as_u64() .ok_or("rawStoredBytes is not a number")? as usize; if escaped.len() != raw_stored { - return Err(format!( - "escaped length {} want rawStoredBytes {raw_stored}", - escaped.len() - )); + return Err(assertion_mismatch(raw_stored, escaped.len())); } } if let Some(escaped_hex) = vector.get("escapedHex").and_then(Value::as_str) { if escaped.bytes != unhex(escaped_hex)? { - return Err(format!( - "escaped bytes {} want {escaped_hex}", - hex::encode(&escaped.bytes) - )); + return Err(assertion_mismatch(escaped_hex, hex::encode(&escaped.bytes))); } } let decoded = decompress_payload(got.payload.clone(), MAX_DECOMPRESSED_BYTES); if decoded.payload != raw { - return Err(format!("compression changed value: {decoded:?}")); + return Err(assertion_mismatch(&raw, &decoded)); } if got.outcome == CompressionOutcome::Compressed { if !got.payload.binary || got.stored_bytes >= escaped.len() || got.payload.len() != got.stored_bytes { - return Err(format!("compression grew or misreported: {got:?}")); + return Err(assertion_mismatch( + "binary payload shorter than escaped raw with accurate stored size", + &got, + )); } let mut marker = if raw.binary { MARKER_ZSTD_BINARY @@ -361,13 +366,10 @@ pub fn check_compression_write_vector(vector: &Value) -> Result<(), String> { .map_err(|_| "modeled marker is not a byte".to_string())?; } if got.payload.bytes.first() != Some(&marker) { - return Err(format!( - "compressed payload marker {:?} want {marker}", - got.payload.bytes.first() - )); + return Err(assertion_mismatch(marker, got.payload.bytes.first())); } } else if got.payload != escaped { - return Err(format!("raw representation differs: {got:?}")); + return Err(assertion_mismatch(&escaped, &got)); } Ok(()) } diff --git a/rust/tests/formal/key_vectors.rs b/rust/tests/formal/key_vectors.rs index 744ad44d..7575b373 100644 --- a/rust/tests/formal/key_vectors.rs +++ b/rust/tests/formal/key_vectors.rs @@ -12,7 +12,7 @@ use std::collections::BTreeMap; use dialcache::identity::{cohort, cohort_hash, normalize_args, ArgValue, Identity}; use serde_json::Value; -use super::fixtures::LONE_SURROGATE_MARKER; +use super::fixtures::{assertion_mismatch, LONE_SURROGATE_MARKER}; fn identity_from(value: &Value) -> Result { serde_json::from_value(value.clone()) @@ -46,26 +46,20 @@ fn expected_str(value: &Value, field: &str) -> Result, String> { /// `keyVectors`: `keys()` must succeed and equal `logicalKey`, `valueKey` and `watermarkKey`. pub fn check_key_vector(vector: &Value) -> Result<(), String> { let identity = identity_from(&vector["input"])?; - let keys = identity - .keys() - .map_err(|error| format!("keys rejected a valid identity: {error}"))?; let logical = expected_str(vector, "logicalKey")?.ok_or("logicalKey missing")?; let value = expected_str(vector, "valueKey")?.ok_or("valueKey missing")?; let watermark = expected_str(vector, "watermarkKey")?; + let keys = identity + .keys() + .map_err(|error| assertion_mismatch((&logical, &value, &watermark), error))?; if keys.logical != logical { - return Err(format!( - "logical key {:?} != expected {logical:?}", - keys.logical - )); + return Err(assertion_mismatch(logical, keys.logical)); } if keys.value != value { - return Err(format!("value key {:?} != expected {value:?}", keys.value)); + return Err(assertion_mismatch(value, keys.value)); } if keys.watermark != watermark { - return Err(format!( - "watermark key {:?} != expected {watermark:?}", - keys.watermark - )); + return Err(assertion_mismatch(watermark, keys.watermark)); } Ok(()) } @@ -138,12 +132,15 @@ pub fn check_invalid_key_vector(vector: &Value) -> Result<(), String> { let unrepresentable = components(&identity).any(|component| component.contains(LONE_SURROGATE_MARKER)); if !unrepresentable && identity.keys().is_ok() { - return Err("invalid key accepted".to_owned()); + return Err(assertion_mismatch("invalid key rejected", "accepted")); } if let Some(raw) = vector.get("inputUtf16") { if let Some(rebuilt) = identity_from_utf16(raw, identity.tracked)? { if rebuilt.keys().is_ok() { - return Err("invalid key rebuilt from UTF-16 units accepted".to_owned()); + return Err(assertion_mismatch( + "invalid UTF-16 key rejected", + "accepted", + )); } } } @@ -204,10 +201,9 @@ pub fn check_normalize_args_vector(vector: &Value) -> Result<(), String> { } let expected: Vec<(String, String)> = serde_json::from_value(vector["expected"].clone()) .map_err(|error| format!("expected pairs do not parse: {error}"))?; - let actual = - normalize_args(record).map_err(|error| format!("normalize_args failed: {error}"))?; + let actual = normalize_args(record).map_err(|error| assertion_mismatch(&expected, error))?; if actual != expected { - return Err(format!("normalized {actual:?} != expected {expected:?}")); + return Err(assertion_mismatch(expected, actual)); } Ok(()) } @@ -216,22 +212,27 @@ pub fn check_normalize_args_vector(vector: &Value) -> Result<(), String> { /// exactly, and the FNV-1a numerator must equal `hashNumerator` when present. pub fn check_ramp_vector(vector: &Value) -> Result<(), String> { let identity = identity_from(&vector["input"])?; - let keys = identity - .keys() - .map_err(|error| format!("keys rejected the ramp identity: {error}"))?; let layer = vector["layer"].as_str().ok_or("layer missing")?; let sample = vector["sample"].as_f64().ok_or("sample missing")?; + let numerator = vector + .get("hashNumerator") + .map(|value| { + value + .as_u64() + .ok_or("hashNumerator is not an unsigned integer") + }) + .transpose()?; + let keys = identity + .keys() + .map_err(|error| assertion_mismatch("valid rollout identity", error))?; let actual = cohort(&keys.logical, layer); if actual != sample { - return Err(format!("cohort {actual:.17} != expected {sample:.17}")); + return Err(assertion_mismatch(sample, actual)); } - if let Some(numerator) = vector.get("hashNumerator") { - let numerator = numerator - .as_u64() - .ok_or("hashNumerator is not an unsigned integer")?; + if let Some(numerator) = numerator { let hash = u64::from(cohort_hash(&keys.logical, layer)); if hash != numerator { - return Err(format!("hash numerator {hash} != expected {numerator}")); + return Err(assertion_mismatch(numerator, hash)); } } Ok(()) diff --git a/rust/tests/formal/mod.rs b/rust/tests/formal/mod.rs index f4820325..d7c2111b 100644 --- a/rust/tests/formal/mod.rs +++ b/rust/tests/formal/mod.rs @@ -8,6 +8,7 @@ //! per binary. #![allow(dead_code)] +pub mod causal; pub mod core_driver; pub mod driver; pub mod gate; diff --git a/rust/tests/harness_infra.rs b/rust/tests/harness_infra.rs index c9ba9ae3..4a837d30 100644 --- a/rust/tests/harness_infra.rs +++ b/rust/tests/harness_infra.rs @@ -93,6 +93,256 @@ fn without(base: &Value, field: &str) -> Value { Value::Object(changed) } +#[test] +fn publication_causality_distinguishes_overlapping_source_owners_and_deadlines() { + use formal::causal::{assert_publication_causality, CausalEvent::*}; + let overlap = vec![ + SourceStart { + id: 0, + owner: 10, + at_ms: 0, + budget_ms: Some(10), + }, + SourceStart { + id: 1, + owner: 20, + at_ms: 10, + budget_ms: Some(10), + }, + SourceSettlement { + id: 0, + at_ms: 15, + outcome: "resolve", + }, + SourceSettlement { + id: 1, + at_ms: 16, + outcome: "resolve", + }, + ]; + for (source, owner, expected) in [ + (0, 10, "late raw settlement"), + (1, 10, "different invocation"), + ] { + let mut history = overlap.clone(); + history.push(WriteDispatch { + source: Some(source), + owner: Some(owner), + at_ms: 17, + }); + let error = assert_publication_causality(&history).unwrap_err(); + assert!(error.starts_with("CAUSAL_PROPERTY_FAILURE"), "{error}"); + assert!(error.contains(expected), "{error}"); + } + let mut accepted = overlap.clone(); + accepted.push(WriteDispatch { + source: Some(1), + owner: Some(20), + at_ms: 30, + }); + assert_publication_causality(&accepted) + .expect("publication can outlive an accepted source's deadline"); + assert_publication_causality(&overlap[..2]).expect("pending prefixes are allowed"); + assert_publication_causality(&[ + SourceStart { + id: 0, + owner: 0, + at_ms: 0, + budget_ms: None, + }, + SourceSettlement { + id: 0, + at_ms: 100_000, + outcome: "resolve", + }, + WriteDispatch { + source: Some(0), + owner: Some(0), + at_ms: 100_000, + }, + ]) + .expect("unbounded sources stay unbounded"); + for outcome in [None, Some("reject")] { + let mut history = vec![SourceStart { + id: 0, + owner: 0, + at_ms: 0, + budget_ms: Some(10), + }]; + if let Some(outcome) = outcome { + history.push(SourceSettlement { + id: 0, + at_ms: 1, + outcome, + }); + } + history.push(WriteDispatch { + source: Some(0), + owner: Some(0), + at_ms: 1, + }); + assert!(assert_publication_causality(&history) + .unwrap_err() + .contains("exact source's successful settlement")); + } +} + +#[test] +fn malformed_monitor_inputs_never_become_mutation_evidence() { + use formal::causal::{assert_publication_causality, CausalEvent::*}; + use formal::driver::{Driver, HistoryEvent}; + for history in [ + vec![SourceStart { + id: 0, + owner: 0, + at_ms: 0, + budget_ms: Some(0), + }], + vec![SourceSettlement { + id: 0, + at_ms: 0, + outcome: "resolve", + }], + vec![WriteDispatch { + source: None, + owner: None, + at_ms: 0, + }], + vec![SourceStart { + id: 0, + owner: 0, + at_ms: -1, + budget_ms: None, + }], + vec![ + SourceStart { + id: 0, + owner: 0, + at_ms: 0, + budget_ms: None, + }, + SourceSettlement { + id: 0, + at_ms: 1, + outcome: "unknown", + }, + ], + ] { + let error = assert_publication_causality(&history).unwrap_err(); + assert!(!error.contains("CAUSAL_PROPERTY_FAILURE"), "{error}"); + } + let start = HistoryEvent { + event: "sourceStart", + id: 0, + at: 0, + outcome: "", + duration_ms: 0.0, + failed: false, + }; + let settled = HistoryEvent { + event: "sourceSettlement", + at: 10, + outcome: "resolve", + ..start.clone() + }; + let completed = HistoryEvent { + event: "fallbackCompletion", + at: 10, + duration_ms: 10.0, + ..start.clone() + }; + for history in [ + vec![completed.clone()], + vec![settled.clone()], + vec![ + start.clone(), + HistoryEvent { + outcome: "unknown", + ..settled.clone() + }, + ], + ] { + let error = Driver::assert_effects_events(&history).unwrap_err(); + assert!(!error.contains("CAUSAL_PROPERTY_FAILURE"), "{error}"); + } + let error = Driver::assert_effects_events(&[start, settled, completed]).unwrap_err(); + assert!( + error.starts_with("CAUSAL_PROPERTY_FAILURE rule=C25 event="), + "{error}" + ); +} + +#[test] +fn invocation_context_follows_spawn_and_defer_without_leaking_between_polls() { + use dialcache::{testing::TestExecutor, Runtime}; + use formal::causal::{current_invocation, InvocationFuture, InvocationRuntime}; + use formal::gate::Gate; + use parking_lot::Mutex; + use std::sync::Arc; + let mut exec = TestExecutor::new(0); + let runtime = Arc::new(InvocationRuntime(exec.runtime.clone())); + let seen = Arc::new(Mutex::new(Vec::new())); + let gate = Gate::<()>::new(); + for owner in [10, 20] { + let runtime = runtime.clone(); + let seen = seen.clone(); + let gate = gate.clone(); + exec.spawn(InvocationFuture::new(Some(owner), async move { + seen.lock().push(current_invocation()); + let deferred_seen = seen.clone(); + runtime.defer(Box::pin(async move { + deferred_seen.lock().push(current_invocation()); + })); + runtime.spawn(Box::pin(async move { + gate.wait().await; + seen.lock().push(current_invocation()); + })); + })); + } + exec.drain(); + assert_eq!(current_invocation(), None); + gate.settle(()); + exec.drain(); + assert_eq!(current_invocation(), None); + let mut seen = seen.lock().clone(); + seen.sort(); + assert_eq!( + seen, + [Some(10), Some(10), Some(10), Some(20), Some(20), Some(20)] + ); + let panicking = InvocationFuture::new(Some(30), async { panic!("controlled context unwind") }); + assert!(std::panic::catch_unwind(std::panic::AssertUnwindSafe( + || futures::executor::block_on(panicking) + )) + .is_err()); + assert_eq!(current_invocation(), None); +} + +#[test] +fn behavior_driver_attributes_detached_writes_to_their_actual_sources() { + use formal::causal::CausalEvent; + use formal::driver::Driver; + let mut driver = Driver::new(json!({"policy":{"ttlSec":{"remote":10},"coalesce":false}})); + for input in [ + json!({"op":"begin", "key":"same"}), + json!({"op":"begin", "key":"same"}), + json!({"op":"resolve", "loader":1, "value":2}), + json!({"op":"resolve", "loader":0, "value":1}), + ] { + driver.apply(&input).unwrap(); + } + let writes: Vec<_> = driver + .causal_history() + .into_iter() + .filter_map(|event| match event { + CausalEvent::WriteDispatch { source, owner, .. } => Some((source, owner)), + _ => None, + }) + .collect(); + assert_eq!(writes, [(Some(1), Some(1)), (Some(0), Some(0))]); + driver.close(); +} + fn prepared_control() -> Prepared { Prepared { session: "1".to_string(), diff --git a/rust/tests/local_storage.rs b/rust/tests/local_storage.rs new file mode 100644 index 00000000..f7ab41b8 --- /dev/null +++ b/rust/tests/local_storage.rs @@ -0,0 +1,73 @@ +//! Native allocation and eviction guarantees of the default local store. + +use std::num::NonZeroUsize; +use std::sync::Arc; + +use dialcache::limits::MAX_SAFE_INTEGER; +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{ + DialCache, Identity, LocalEntry, LocalRead, LocalStore, LruLocalStore, Operation, Policy, +}; + +#[test] +fn maximum_supported_capacity_allocates_only_as_entries_arrive() { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let capacity = usize::try_from(MAX_SAFE_INTEGER).unwrap_or(usize::MAX); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .local_capacity(capacity) + .build() + .expect("the maximum supported capacity must remain constructible"); + let (first, second) = executor.block_on(async move { + let request = cache.enable_guard(); + let operation = Operation::::new(Identity::new("thing", "one", "LargeCapacity")) + .policy(Policy::default().local_ttl_sec(60)); + let first = cache + .get_or_load(request.scope(), operation.clone(), |_| async { Ok(7) }) + .await + .expect("source success"); + let second = cache + .get_or_load(request.scope(), operation, |_| async { Ok(8) }) + .await + .expect("local hit"); + (first, second) + }); + assert_eq!((*first, *second), (7, 7)); +} + +fn entry(value: u64) -> LocalEntry { + LocalEntry { + value: Arc::new(value), + inserted_ms: 0, + ttl_ms: 1_000, + } +} + +#[test] +fn sparse_storage_preserves_capacity_and_lru_eviction() { + let mut store = LruLocalStore::new(NonZeroUsize::new(2).unwrap()); + assert!(store.put("a".into(), entry(1)).unwrap().is_none()); + assert!(store.put("b".into(), entry(2)).unwrap().is_none()); + assert!(matches!(store.get("a", 0).unwrap(), LocalRead::Live(_))); + + let evicted = store + .put("c".into(), entry(3)) + .unwrap() + .expect("at capacity"); + assert_eq!(evicted.value.downcast_ref::(), Some(&2)); + assert_eq!(store.len(), 2); + assert!(matches!(store.get("b", 0).unwrap(), LocalRead::Absent)); + + let replaced = store + .put("a".into(), entry(4)) + .unwrap() + .expect("replacement"); + assert_eq!(replaced.value.downcast_ref::(), Some(&1)); + let evicted = store + .put("d".into(), entry(5)) + .unwrap() + .expect("at capacity"); + assert_eq!(evicted.value.downcast_ref::(), Some(&3)); + assert_eq!(store.len(), 2); +} diff --git a/rust/tests/protocol_frames.rs b/rust/tests/protocol_frames.rs index 1e786de7..386f4406 100644 --- a/rust/tests/protocol_frames.rs +++ b/rust/tests/protocol_frames.rs @@ -8,10 +8,24 @@ mod fixtures; #[path = "formal/frame_vectors.rs"] mod frame_vectors; -use serde_json::Value; +use serde_json::{json, Value}; type Check = fn(&Value) -> Result<(), String>; +#[test] +fn frame_assertions_distinguish_library_rejections_from_bad_fixtures() { + let valid_shape = json!({"createdAtMs":0, "payloadType":"string", "payloadUtf8":"value", "frameHex":"expected"}); + let error = frame_vectors::check_frame_vector(&valid_shape).unwrap_err(); + assert!( + error.starts_with("PROTOCOL_ASSERTION_FAILURE expected:"), + "{error}" + ); + let mut malformed = valid_shape; + malformed["frameHex"] = json!(5); + let error = frame_vectors::check_frame_vector(&malformed).unwrap_err(); + assert!(!error.contains("PROTOCOL_ASSERTION_FAILURE"), "{error}"); +} + /// The eight groups this test owns, with their fixed-corpus sizes. const GROUPS: [(&str, usize, Check); 8] = [ ("frameVectors", 8, frame_vectors::check_frame_vector), diff --git a/rust/tests/protocol_keys.rs b/rust/tests/protocol_keys.rs index 6b14551b..3914b3f1 100644 --- a/rust/tests/protocol_keys.rs +++ b/rust/tests/protocol_keys.rs @@ -9,10 +9,27 @@ mod fixtures; #[path = "formal/key_vectors.rs"] mod key_vectors; -use serde_json::Value; +use serde_json::{json, Value}; type Check = fn(&Value) -> Result<(), String>; +#[test] +fn key_assertions_distinguish_library_rejections_from_bad_fixtures() { + let valid_shape = json!({ + "input":{"namespace":"bad{namespace", "keyType":"id", "id":"1", "useCase":"Example", "tracked":false, "args":[]}, + "logicalKey":"expected", "valueKey":"expected", "watermarkKey":null, + }); + let error = key_vectors::check_key_vector(&valid_shape).unwrap_err(); + assert!( + error.starts_with("PROTOCOL_ASSERTION_FAILURE expected:"), + "{error}" + ); + let mut malformed = valid_shape; + malformed["logicalKey"] = json!(5); + let error = key_vectors::check_key_vector(&malformed).unwrap_err(); + assert!(!error.contains("PROTOCOL_ASSERTION_FAILURE"), "{error}"); +} + const GROUPS: [(&str, Check); 4] = [ ("keyVectors", key_vectors::check_key_vector), ("invalidKeyVectors", key_vectors::check_invalid_key_vector), diff --git a/rust/tests/redis_integration.rs b/rust/tests/redis_integration.rs index c160b640..ff28a326 100644 --- a/rust/tests/redis_integration.rs +++ b/rust/tests/redis_integration.rs @@ -3,7 +3,8 @@ //! Valkey 8 standalone servers plus a single-node Redis 7 cluster, each //! exercised for complete-frame round trips, watermark fencing, the //! `EVALSHA` to `EVAL` recovery after `SCRIPT FLUSH`, and the full -//! invalidation vector corpus (`formal/invalidation_vectors.rs`); and a +//! invalidation vector corpus (`formal/invalidation_vectors.rs`), plus actual +//! TypeScript/Rust key, payload and invalidation interoperability; and a //! replicated Redis 7 cluster that pins tracked reads to the slot primary. //! //! Every test here is `#[ignore]`d: `cargo test` reports it as ignored, and @@ -33,6 +34,7 @@ mod formal; #[path = "formal/invalidation_vectors.rs"] mod invalidation_vectors; +mod redis_interop; use std::collections::HashMap; use std::net::TcpListener; @@ -453,9 +455,10 @@ async fn round_trip(adapter: &RedisAdapter, connection: & ); } -async fn exercise(connection: C, label: &str) { +async fn exercise(connection: C, label: &str, endpoint: &str, cluster: bool) { let adapter = RedisAdapter::new(connection); round_trip(&adapter, adapter.connection(), label).await; + redis_interop::exercise(&adapter, endpoint, cluster, label).await; let vectors = invalidation_vectors::load_corpus(&repo_root(), sha256_hex); let prefix = format!("{{rust-invalidation-{label}}}:"); match invalidation_vectors::replay_all(&adapter, adapter.connection(), &prefix, &vectors).await @@ -473,8 +476,8 @@ async fn exercise(connection: C, label: &str) { fn run_standalone(test: &str, image: &str) { let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); runtime.block_on(async { - let (_container, connection) = standalone(image).await; - exercise(connection, test).await; + let (container, connection) = standalone(image).await; + exercise(connection, test, &published_endpoint(&container), false).await; }); } @@ -501,8 +504,8 @@ fn valkey_8_standalone() { fn redis_7_single_node_cluster() { let runtime = tokio::runtime::Runtime::new().expect("tokio runtime"); runtime.block_on(async { - let (_container, connection) = single_node_cluster("redis:7-alpine").await; - exercise(connection, "cluster").await; + let (container, connection) = single_node_cluster("redis:7-alpine").await; + exercise(connection, "cluster", &published_endpoint(&container), true).await; }); } diff --git a/rust/tests/redis_interop/mod.rs b/rust/tests/redis_interop/mod.rs new file mode 100644 index 00000000..6fe60f2e --- /dev/null +++ b/rust/tests/redis_interop/mod.rs @@ -0,0 +1,429 @@ +//! Bidirectional production TypeScript/Rust reads, writes and invalidation on +//! the same server. Neither language receives the other's derived key/bytes. + +use std::fs::{self, File}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +use dialcache::identity::{normalize_args, ArgValue, Identity, Keys}; +use dialcache::protocol::{ + compress_payload, decompress_payload, escape_raw_payload, CompressionConfig, +}; +use dialcache::redis::{RedisAdapter, RedisConnection}; +use dialcache::{ + Frame, InvalidateRequest, JsonCodec, MissReason, Payload, ReadRequest, ReadResult, Remote, + WriteRequest, +}; +use serde_json::{json, Value}; + +const STAMP: u64 = 1_700_000_000_000; +const MAX_DECOMPRESSED_BYTES: usize = 512 * 1024 * 1024; + +struct TypeScript { + directory: PathBuf, + root: PathBuf, +} + +impl TypeScript { + fn new() -> Self { + let root = std::env::var_os("DIALCACHE_TS_ROOT") + .map(PathBuf::from) + .unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("..")); + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let directory = std::env::temp_dir().join(format!( + "dialcache-rust-interop-{}-{nonce}", + std::process::id() + )); + fs::create_dir(&directory).expect("create TypeScript bundle directory"); + let script = Self { directory, root }; + // Use tsup's declared esbuild dependency, exactly as the Go integration + // does, and import current production TypeScript rather than a fixture codec. + let build = "const {createRequire}=require('node:module');const {buildSync}=createRequire(require.resolve('tsup'))('esbuild');buildSync({entryPoints:[process.argv[1]],outfile:process.argv[2],bundle:true,platform:'node',format:'cjs'});"; + let output = Command::new("node") + .arg("-e") + .arg(build) + .arg(script.root.join("go/redis_interop.ts")) + .arg(script.directory.join("interop.cjs")) + .current_dir(&script.root) + .output() + .expect("bundle TypeScript interop"); + assert!( + output.status.success(), + "TypeScript bundle failed: {}", + String::from_utf8_lossy(&output.stderr) + ); + script + } + + fn run(&self, endpoint: &str, cluster: bool, actions: &[Value]) -> Vec { + let input = self.directory.join("input.json"); + let output = self.directory.join("output.json"); + let error = self.directory.join("stderr.txt"); + fs::write( + &input, + serde_json::to_vec( + &json!({ "endpoint": endpoint, "cluster": cluster, "actions": actions }), + ) + .unwrap(), + ) + .unwrap(); + // Files keep large key-probe output from filling an unread pipe while + // the parent waits, and let the timeout kill and reap a stuck Node child. + let mut child = Command::new("node") + .arg(self.directory.join("interop.cjs")) + .current_dir(&self.root) + .stdin(Stdio::from(File::open(input).unwrap())) + .stdout(Stdio::from(File::create(&output).unwrap())) + .stderr(Stdio::from(File::create(&error).unwrap())) + .spawn() + .expect("start TypeScript interop"); + let deadline = Instant::now() + Duration::from_secs(30); + let status = loop { + if let Some(status) = child.try_wait().expect("poll TypeScript interop") { + break status; + } + if Instant::now() >= deadline { + let _ = child.kill(); + let _ = child.wait(); + panic!( + "TypeScript interop exceeded 30 seconds: {}", + fs::read_to_string(&error).unwrap_or_default() + ); + } + std::thread::sleep(Duration::from_millis(10)); + }; + assert!( + status.success(), + "TypeScript interop failed: {}", + fs::read_to_string(error).unwrap_or_default() + ); + let results: Vec = + serde_json::from_slice(&fs::read(output).unwrap()).expect("TypeScript JSON results"); + assert_eq!( + results.len(), + actions.len(), + "every interop operation must return an observation" + ); + results + } +} + +impl Drop for TypeScript { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.directory); + } +} + +/// Both ports receive the same scalar inputs and independently normalize them. +/// The exact double is transported as bits, not either port's decimal spelling. +fn identity(label: &str, name: &str, tracked: bool, number_bits: u64) -> (Value, Keys) { + let namespace = format!("rust-ts-{label}"); + let args = normalize_args([ + ("case", ArgValue::Str(name.to_string())), + ("number", ArgValue::Number(f64::from_bits(number_bits))), + ("\u{e000}", ArgValue::Bool(false)), + ("😀", ArgValue::Null), + ]) + .unwrap(); + let keys = Identity::new("entity é", "shared/id", "lookup?#") + .namespace(&namespace) + .tracked(tracked) + .args(args) + .keys() + .unwrap(); + let input = json!({ + "namespace": namespace, "keyType": "entity é", "id": "shared/id", "useCase": "lookup?#", + "trackForInvalidation": tracked, + "args": { "case": name, "\u{e000}": false, "😀": null }, + "numberBits": { "number": format!("{number_bits:016x}") }, + }); + (input, keys) +} + +fn key_json(keys: &Keys) -> Value { + json!({ "logical": keys.logical, "value": keys.value, "watermark": keys.watermark }) +} + +fn read_request(keys: &Keys) -> ReadRequest { + ReadRequest { + value_key: keys.value.clone(), + watermark_key: keys.watermark.clone(), + } +} + +fn prepare(payload: Payload, compressed: bool) -> Payload { + if compressed { + compress_payload( + payload, + &CompressionConfig { + threshold_bytes: 1, + level: 3, + }, + MAX_DECOMPRESSED_BYTES, + ) + .unwrap() + .payload + } else { + escape_raw_payload(payload) + } +} + +struct Case { + identity: Value, + keys: Keys, + payload: Payload, + value: Option, + compressed: bool, +} + +pub async fn exercise( + adapter: &RedisAdapter, + endpoint: &str, + cluster: bool, + label: &str, +) { + let script = TypeScript::new(); + // Native Number::toString is explicitly outside Quint's integer model. + // Probe the real TS key builder with regression ties, exponent boundaries, + // special values and deterministic whole-domain IEEE754 samples. + let mut bits: Vec = [ + 0.0_f64, + -0.0, + f64::from_bits(0x430c6bf526340002), + f64::from_bits(0xc30c6bf526340002), + f64::from_bits(0x430c6bf526340006), + f64::from_bits(0xc30c6bf526340006), + f64::from_bits(0x42d6bcc41e900008), + f64::from_bits(0x42d6bcc41e900018), + 1e-6, + 1e-7, + 1e20, + 1e21, + f64::MIN_POSITIVE, + f64::MAX, + f64::INFINITY, + f64::NEG_INFINITY, + f64::NAN, + ] + .into_iter() + .map(f64::to_bits) + .collect(); + bits.push(1); // Smallest positive subnormal. + let mut sample = 88_172_645_463_325_252_u64; + for _ in 0..512 { + sample ^= sample << 13; + sample ^= sample >> 7; + sample ^= sample << 17; + bits.push(sample); + } + let probes: Vec<_> = bits + .into_iter() + .enumerate() + .map(|(index, bits)| identity(label, "number-probe", index % 2 == 0, bits)) + .collect(); + let actions: Vec<_> = probes + .iter() + .map(|(identity, _)| json!({ "op": "key", "identity": identity })) + .collect(); + for ((_, expected), actual) in probes.iter().zip(script.run(endpoint, cluster, &actions)) { + assert_eq!( + actual["keys"], + key_json(expected), + "{label}: independently constructed numeric keys" + ); + } + + let values = [ + Value::Null, + json!(false), + json!(0), + json!(""), + json!({"id": "cross-language", "nested": [1, null, false, "é😀"]}), + json!("DialCache é😀".repeat(1000)), + ]; + let binaries = [ + vec![], + vec![0, 1, 2, 255, 0xe2, 0x82], + vec![1, 0xff], + vec![2, 0xff], + vec![3, 0xff], + [0, 1, 2, 255, 0xe2, 0x82].repeat(1000), + ]; + let mut cases = Vec::new(); + for tracked in [false, true] { + for (index, value) in values.iter().enumerate() { + let (identity, keys) = + identity(label, &format!("json-{index}"), tracked, 0x430c6bf526340002); + cases.push(Case { + identity, + keys, + payload: JsonCodec::encode_value(value).unwrap(), + value: Some(value.clone()), + compressed: index == values.len() - 1, + }); + } + for (index, bytes) in binaries.iter().enumerate() { + let (identity, keys) = identity( + label, + &format!("binary-{index}"), + tracked, + 0xc30c6bf526340002, + ); + cases.push(Case { + identity, + keys, + payload: Payload::binary(bytes.clone()), + value: None, + compressed: index == binaries.len() - 1, + }); + } + } + for case in &cases { + adapter + .write(WriteRequest { + value_key: case.keys.value.clone(), + frame: Frame { + created_at_ms: STAMP, + payload: prepare(case.payload.clone(), case.compressed), + }, + ttl_ms: 60_000, + }) + .await + .expect("Rust writes interop frame"); + } + let reads: Vec<_> = cases.iter().map(|case| json!({ "op": "read", "identity": case.identity, "binary": case.value.is_none() })).collect(); + let results = script.run(endpoint, cluster, &reads); + for (case, result) in cases.iter().zip(results) { + assert_eq!(result["kind"], "hit", "{label}: TS reads Rust frame"); + assert_eq!( + result["stamp"], STAMP, + "{label}: TS preserves Rust timestamp" + ); + assert_eq!( + result["keys"], + key_json(&case.keys), + "{label}: TS derived the same keys" + ); + if let Some(value) = &case.value { + assert_eq!(&result["value"], value, "{label}: TS decodes Rust JSON"); + } else { + assert_eq!( + result["binaryHex"], + hex::encode(&case.payload.bytes), + "{label}: TS decodes Rust binary" + ); + } + } + let writes: Vec<_> = cases.iter().map(|case| { + let mut action = json!({ "op": "write", "identity": case.identity, "stamp": STAMP + 1, "compress": case.compressed }); + if let Some(value) = &case.value { action["value"] = value.clone(); } + else { action["binaryHex"] = json!(hex::encode(&case.payload.bytes)); } + action + }).collect(); + for (case, result) in cases.iter().zip(script.run(endpoint, cluster, &writes)) { + assert_eq!(result["kind"], "written"); + assert_eq!(result["keys"], key_json(&case.keys)); + } + for case in &cases { + let result = adapter + .read(read_request(&case.keys), super::context()) + .await + .expect("Rust reads TypeScript frame"); + let ReadResult::Hit(frame) = result else { + panic!("{label}: Rust missed TypeScript frame: {result:?}"); + }; + assert_eq!( + frame.created_at_ms, + STAMP + 1, + "{label}: must read the TypeScript overwrite" + ); + let payload = decompress_payload(frame.payload, MAX_DECOMPRESSED_BYTES).payload; + if let Some(expected) = &case.value { + assert_eq!( + &JsonCodec::decode_value::(&payload).unwrap(), + expected, + "{label}: Rust decodes TypeScript JSON" + ); + } else { + assert_eq!( + payload, case.payload, + "{label}: Rust decodes TypeScript binary" + ); + } + } + + // Rust's documented undefined adaptation is Option::None, distinct from + // writing its own null above (which TypeScript must continue to read as null). + let (undefined, undefined_keys) = identity(label, "undefined", true, 0); + script.run( + endpoint, + cluster, + &[json!({ "op": "write", "identity": undefined, "stamp": STAMP, "absent": true })], + ); + let ReadResult::Hit(frame) = adapter + .read(read_request(&undefined_keys), super::context()) + .await + .unwrap() + else { + panic!("Rust must read the TypeScript undefined sentinel"); + }; + assert_eq!( + JsonCodec::decode_value::>(&frame.payload).unwrap(), + None + ); + + let tracked = cases + .iter() + .find(|case| case.keys.watermark.is_some()) + .unwrap(); + script.run(endpoint, cluster, &[json!({ "op": "invalidate", "identity": tracked.identity, "stamp": STAMP + 1, "futureMs": 100 })]); + assert_eq!( + adapter + .read(read_request(&tracked.keys), super::context()) + .await + .unwrap(), + ReadResult::Miss { + reason: MissReason::WatermarkFenced, + observed_watermark_ms: Some(STAMP + 101) + }, + "{label}: TypeScript invalidation fences Rust reads" + ); + adapter + .write(WriteRequest { + value_key: tracked.keys.value.clone(), + frame: Frame { + created_at_ms: STAMP + 102, + payload: JsonCodec::encode_value(&json!("after invalidation")).unwrap(), + }, + ttl_ms: 60_000, + }) + .await + .unwrap(); + let read = json!({ "op": "read", "identity": tracked.identity }); + let result = script.run(endpoint, cluster, std::slice::from_ref(&read)); + assert_eq!( + result[0]["kind"], "hit", + "{label}: TypeScript reads Rust's newer frame" + ); + assert_eq!(result[0]["value"], "after invalidation"); + adapter + .invalidate(InvalidateRequest { + watermark_key: tracked.keys.watermark.clone().unwrap(), + invalidated_at_ms: STAMP + 1, + future_buffer_ms: 200, + }) + .await + .unwrap(); + let result = script.run(endpoint, cluster, &[read]); + assert_eq!( + result[0]["kind"], "miss", + "{label}: Rust invalidation fences TypeScript reads" + ); + assert_eq!(result[0]["reason"], "watermark_fenced"); + assert_eq!(result[0]["observedWatermarkMs"], STAMP + 201); + println!("{label}: {} independent key probes and {} payloads read in both languages, plus both invalidation directions", probes.len(), cases.len()); +} diff --git a/rust/tests/runtime_policy.rs b/rust/tests/runtime_policy.rs new file mode 100644 index 00000000..528a5d7f --- /dev/null +++ b/rust/tests/runtime_policy.rs @@ -0,0 +1,72 @@ +//! Native typed-policy conversion preserves the portable sparse-overlay rules. + +#![cfg(feature = "tokio")] + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use dialcache::{DialCache, Identity, Operation, Policy}; + +fn cache_with_ttl_overlay() -> DialCache { + DialCache::builder() + .local_capacity(0) + .policy_provider(|_| async { Ok(Some(Policy::default().local_ttl_sec(9).into())) }) + .build() + .unwrap() +} + +#[tokio::test] +async fn typed_ttl_overlay_preserves_request_memoization() { + let cache = cache_with_ttl_overlay(); + let request = cache.enable_guard(); + let calls = Arc::new(AtomicUsize::new(0)); + for _ in 0..2 { + let calls = calls.clone(); + let value = cache + .get_or_load( + request.scope(), + Operation::::new(Identity::new("item", "one", "lookup")) + .policy(Policy::default().request_local(true)), + move |_| { + let value = calls.fetch_add(1, Ordering::SeqCst) + 1; + async move { Ok(value) } + }, + ) + .await + .unwrap(); + assert_eq!(*value, 1, "the second call must use the request memo"); + } + assert_eq!(calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn typed_ttl_overlay_preserves_independent_sources() { + let cache = cache_with_ttl_overlay(); + let request = cache.enable_guard(); + let calls = Arc::new(AtomicUsize::new(0)); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let operation = Operation::::new(Identity::new("item", "one", "lookup")) + .policy(Policy::default().coalesce(false)); + let load = { + let calls = calls.clone(); + move |_| { + let calls = calls.clone(); + let barrier = barrier.clone(); + async move { + let value = calls.fetch_add(1, Ordering::SeqCst) + 1; + barrier.wait().await; + Ok(value) + } + } + }; + let first = cache.get_or_load(request.scope(), operation.clone(), load.clone()); + let second = cache.get_or_load(request.scope(), operation, load); + let (first, second) = tokio::time::timeout(Duration::from_secs(5), async { + tokio::join!(first, second) + }) + .await + .expect("a TTL-only overlay must not coalesce independent sources"); + assert_ne!(*first.unwrap(), *second.unwrap()); + assert_eq!(calls.load(Ordering::SeqCst), 2); +} diff --git a/test/formal-rust-semantic-runner.test.ts b/test/formal-rust-semantic-runner.test.ts index 6e57d7d6..dbc10c22 100644 --- a/test/formal-rust-semantic-runner.test.ts +++ b/test/formal-rust-semantic-runner.test.ts @@ -2,10 +2,10 @@ import { readFileSync } from "node:fs"; import { describe, expect, it } from "vitest"; const moduleUrl = new URL("../formal/measure-rust-semantics.mjs", import.meta.url).href; -type Cohort = { state: string; passed: number; failed: number; failingTests: string[]; executedTests: string[] }; +type Cohort = { state: string; passed: number; failed: number; failingTests: string[]; executedTests: string[]; assertionKinds: Record; assertionEvidence: Record }; const { evaluateCargoTestOutput, evaluateRustReport, infrastructureTestFile } = await import(moduleUrl) as { evaluateCargoTestOutput(output: string, exitCode: number, expectedBinaries?: number): Cohort; - evaluateRustReport(text: string, exitCode: number): Cohort; + evaluateRustReport(text: string, exitCode: number, stderr?: string): Cohort; infrastructureTestFile: RegExp; }; @@ -57,11 +57,11 @@ describe("Rust ordinary cohort evaluation", () => { }); }); -function report(cases: [string, string][], finish = true, extra: Record = {}): string { +function report(cases: [string, string, string?][], finish = true, extra: Record = {}): string { const failed = cases.filter(([, status]) => status === "failed").length; const lines = [ JSON.stringify({ schemaVersion: 1, kind: "start", implementation: "rust", startedAt: 1 }), - ...cases.map(([id, status]) => JSON.stringify({ kind: "case", id, status, startedAt: 2, finishedAt: 3, ...(status === "failed" ? { message: "expected 1 got 2" } : {}) })), + ...cases.map(([id, status, message]) => JSON.stringify({ kind: "case", id, status, startedAt: 2, finishedAt: 3, ...(status === "failed" ? { message: message ?? "expected: 1\nactual: 2" } : {}) })), ]; if (finish) lines.push(JSON.stringify({ kind: "finish", status: failed ? "failed" : "passed", finishedAt: 4, cases: cases.length, failed, ...extra })); return lines.join("\n") + "\n"; @@ -80,6 +80,41 @@ describe("Rust harness report evaluation", () => { expect(() => evaluateRustReport(report([["sampled/core/0", "passed"]]), 101)).toThrow(/without a failed case/); }); + it("retains the validated assertion kind and evidence", () => { + const id = "sampled/effects/0"; + const message = 'CAUSAL_PROPERTY_FAILURE rule=C26 event={"index":0,"atMs":0,"event":"writeDispatch","condition":"publication without accepted source success","authorized":false}'; + expect(evaluateRustReport(report([[id, "failed", message]]), 101)).toMatchObject({ + assertionKinds: { [id]: "causal-property" }, assertionEvidence: { [id]: message }, + }); + const vector = "protocol/frameVectors/example"; + const mismatch = "PROTOCOL_ASSERTION_FAILURE expected: 1\nactual: 2"; + expect(evaluateRustReport(report([[vector, "failed", mismatch]]), 101)).toMatchObject({ + assertionKinds: { [vector]: "protocol-assertion" }, assertionEvidence: { [vector]: mismatch }, + }); + }); + + it.each([ + "Malformed replay observation: behaviorObservation at observed.calls", + "malformed next replay command", + "no pending read 0", + "coordinator response is not UTF-8", + "effects contract event 0 fallbackCompletion: fallback completion has no source", + 'CAUSAL_PROPERTY_FAILURE rule=C26 event={"index":0,"atMs":0,"event":"writeDispatch","condition":"publication without accepted source success","authorized":true}', + ])("rejects infrastructure or invalid property evidence: %s", message => { + expect(() => evaluateRustReport(report([["sampled/effects/0", "failed", message]]), 101)).toThrow(/lacks observation or validated causal property evidence/); + }); + + it("rejects global infrastructure failures even when an assertion also failed", () => { + const text = report([["sampled/effects/0", "failed"]]); + expect(() => evaluateRustReport(text, 101, "conformance harness failed: coordinator exited unexpectedly")).toThrow(/infrastructure or coverage/); + expect(() => evaluateRustReport(text, 101, "coverage: effects corpus omits action begin")).toThrow(/infrastructure or coverage/); + }); + + it("rejects failed witness audits and malformed protocol fixtures", () => { + expect(() => evaluateRustReport(report([["witness/effects", "failed", "expected: hash-a actual: hash-b"]]), 101)).toThrow(/witness audit failure/); + expect(() => evaluateRustReport(report([["protocol/frameVectors/example", "failed", "frameHex is neither null nor a string"]]), 101)).toThrow(/protocol failure lacks assertion evidence/); + }); + it("rejects crashed, empty, duplicated or inconsistent reports instead of crediting them", () => { expect(() => evaluateRustReport("", 0)).toThrow(/empty Rust harness report/); expect(() => evaluateRustReport(report([["sampled/core/0", "passed"]], false), 0)).toThrow(/no finish record/); diff --git a/test/formal-validation.test.ts b/test/formal-validation.test.ts index 501b7a6d..01f8706f 100644 --- a/test/formal-validation.test.ts +++ b/test/formal-validation.test.ts @@ -142,11 +142,14 @@ process.exit(Number(process.argv[3] ?? 0));\n`); .rejects.toThrow(/files requiring formatting:\ngo\/cache.go/); }); - it("orders generation and shared witness evaluation before both port replays without duplicate wire generation", () => { + it("orders generation and shared witness evaluation before every port replay without duplicate wire generation", () => { const plan = validationPlan("formal", { directory }); const position = (script: string, argument: string) => plan.findIndex(step => step.args?.[0] === script && step.args.includes(argument)); const tsPrepare = position("formal/conformance.mjs", "typescript"); const goPrepare = position("formal/conformance.mjs", "go"); + const rustPrepare = position("formal/conformance.mjs", "rust"); + const goCompletion = position("formal/conformance.mjs", ".formal-traces/go-completion.json"); + const rustCompletion = position("formal/conformance.mjs", ".formal-traces/rust-completion.json"); const tsCompletion = position("formal/conformance.mjs", ".formal-traces/ts-completion.json"); const witnesses = position("formal/witnesses.mjs", "evaluate"); expect(position("formal/run-models.mjs", "check")).toBeLessThan(position("formal/run-models.mjs", "generate")); @@ -154,6 +157,9 @@ process.exit(Number(process.argv[3] ?? 0));\n`); expect(witnesses).toBeLessThan(tsPrepare); expect(tsPrepare).toBeLessThan(tsCompletion); expect(tsCompletion).toBeLessThan(goPrepare); + expect(goPrepare).toBeLessThan(goCompletion); + expect(goCompletion).toBeLessThan(rustPrepare); + expect(rustPrepare).toBeLessThan(rustCompletion); expect(plan.filter(step => step.args?.[0] === "formal/run-models.mjs" && step.args[1] === "generate")).toHaveLength(1); expect(plan.filter(step => step.args?.[0] === "formal/witnesses.mjs")).toHaveLength(1); expect(plan.some(step => step.args?.[0] === "formal/generate-artifacts.mjs")).toBe(false); @@ -227,6 +233,29 @@ process.exit(Number(process.argv[3] ?? 0));\n`); expect(ts.some(step => step.args?.[0] === "formal/witnesses.mjs" || step.args?.[0] === "formal/run-models.mjs")).toBe(false); }); + it("replays Rust from its crate with full corpus selectors and a dedicated report before the completion gate", () => { + const plan = validationPlan("formal-rust", { directory, environment }); + expect(plan[0]!.remove).toEqual([".formal-traces/rust-completion.json"]); + expect(plan[1]!.args).toEqual(["formal/conformance.mjs", "prepare", "rust", ".formal-traces/rust-context.json"]); + const replay = plan.find(step => step.command === "cargo")!; + expect(replay).toMatchObject({ cwd: "rust", args: ["test", "--release", "--all-features", "--test", "conformance"] }); + expect(replay.env).toEqual({ + DIALCACHE_MBT_TRACE_DIR: join(directory, ".formal-traces/conformance"), + DIALCACHE_EFFECTS_TRACE_DIR: join(directory, ".formal-traces/effects"), + DIALCACHE_FEATURE_TRACE_DIR: join(directory, ".formal-traces/features"), + DIALCACHE_WITNESS_EVIDENCE_DIR: join(directory, ".formal-traces/go-parity-witnesses"), + DIALCACHE_RUST_REPORT: join(directory, ".formal-traces/rust-replay.jsonl"), + }); + expect(plan.slice(plan.indexOf(replay) + 1).map(step => step.args)).toEqual([ + ["formal/check-rust-replay.mjs"], + ["formal/conformance-adapters.mjs", "rust", ".formal-traces/rust-replay.jsonl", ".formal-traces/rust-context.json"], + ["formal/conformance.mjs", "check", ".formal-traces/rust-completion.json", ".formal-traces/rust-context.json"], + ]); + const smoke = validationPlan("smoke", { directory }).find(step => step.command === "cargo")!; + expect(smoke.args).toEqual(["test", "--all-features", "--test", "conformance"]); + expect(smoke.env).toBeUndefined(); + }); + it("lets Go parity and every mutation measurement run off the generated corpus without completion checks", () => { const isCompletionCheck = (step: Step) => step.args?.[0] === "formal/conformance.mjs" && step.args[1] === "check"; const go = validationPlan("formal-go", { directory }); @@ -289,14 +318,16 @@ process.exit(Number(process.argv[3] ?? 0));\n`); for (const target of ["mutations-ts", "mutations-go", "mutations-rust"]) expect(targetDescriptions[target]).toMatch(/MUTATION_ONLY=,/); }); - it("merges each language's shards with a plain Node step that needs neither Quint nor Go", () => { + it("merges each language's shards with a plain Node step that needs no native toolchains or Quint", () => { expect(validationPlan("mutations-merge-ts", { directory })).toEqual([ { label: "Merge TypeScript mutation shards", command: process.execPath, args: ["formal/merge-mutation-reports.mjs", "ts"] }, ]); expect(validationPlan("mutations-merge-go", { directory }).map(step => step.args)).toEqual([["formal/merge-mutation-reports.mjs", "go"]]); + expect(validationPlan("mutations-merge-rust", { directory }).map(step => step.args)).toEqual([["formal/merge-mutation-reports.mjs", "rust"]]); fakeTool("quint", 'console.error("quint: not installed"); process.exit(1)'); fakeTool("go", 'console.error("go: not installed"); process.exit(1)'); - for (const target of ["mutations-merge-ts", "mutations-merge-go"]) { + fakeTool("cargo", 'console.error("cargo: not installed"); process.exit(1)'); + for (const target of ["mutations-merge-ts", "mutations-merge-go", "mutations-merge-rust"]) { expect(() => checkPrerequisites(target, { directory, environment, nodeVersion: "v24.20.0" })).not.toThrow(); expect(targetDescriptions[target]).toMatch(/Merge .* mutation shards/); } @@ -358,7 +389,7 @@ process.exit(Number(process.argv[3] ?? 0));\n`); for (const target of ["mutations-ts", "mutations-go"]) { expect(() => checkPrerequisites(target, { directory, environment, nodeVersion: "v24.20.0" })).toThrow(/docker/); } - for (const target of ["mutations-merge-ts", "mutations-merge-go"]) { + for (const target of ["mutations-merge-ts", "mutations-merge-go", "mutations-merge-rust"]) { expect(() => checkPrerequisites(target, { directory, environment, nodeVersion: "v24.20.0" })).not.toThrow(); } }); @@ -479,6 +510,32 @@ describe("full formal workflow shape", () => { expect(gate.run).toMatch(/test "\$GO_MUTATIONS_MERGE_RESULT" = success/); }); + it("requires Rust replay and merged mutations and retains Rust completion in the summary", () => { + const parity = jobs["rust-parity"]!; + expect(parity.steps.find(step => step.uses === "./.github/actions/setup-validation")!.with).toEqual({ rust: "true" }); + expect(parity.steps.map(step => step.run).filter(Boolean)).toEqual(["make formal-rust"]); + const mutations = jobs["rust-mutations"]!; + const shards = mutations.strategy!.matrix!.shard as number[]; + expect(mutations.env).toEqual({ MUTATION_SHARD: "$" + "{{ matrix.shard }}/" + shards.length }); + expect(mutations.steps.map(step => step.run).filter(Boolean)).toEqual(["make mutations-rust"]); + const merge = jobs["rust-mutations-merge"]!; + expect(needsOf(merge)).toEqual(["rust-mutations"]); + expect(merge.if).toBe("always() && needs.rust-mutations.result != 'skipped'"); + expect(merge.steps.map(step => step.run).filter(Boolean)).toEqual(["make mutations-merge-rust"]); + const aggregate = jobs["formal-full"]!; + expect(needsOf(aggregate)).toEqual(expect.arrayContaining(["rust-parity", "rust-mutations-merge"])); + expect(needsOf(aggregate)).not.toContain("rust-mutations"); + const gate = aggregate.steps.find(step => step.run?.includes("_RESULT"))!; + expect(gate.env).toMatchObject({ RUST_RESULT: "$" + "{{ needs.rust-parity.result }}", + RUST_MUTATIONS_MERGE_RESULT: "$" + "{{ needs.rust-mutations-merge.result }}" }); + expect(gate.run).toMatch(/test "\$RUST_RESULT" = success/); + expect(gate.run).toMatch(/test "\$RUST_MUTATIONS_MERGE_RESULT" = success/); + const summary = aggregate.steps.find(step => step.uses?.startsWith("actions/upload-artifact"))!.with!; + expect(summary.path).toContain("formal-summary/rust/rust-completion.json"); + expect(summary.path).toContain("formal-summary/rust/rust-context.json"); + expect(summary.path).toContain("formal-summary/rust/rust-replay-summary.json"); + }); + it("requires the model check in the aggregate and retains its report in the long-lived summary", () => { const aggregate = jobs["formal-full"]!; expect(needsOf(aggregate)).toEqual(expect.arrayContaining(["check-models", "generate", "typescript-parity", "go-parity"])); From b3d2829e4aaff1a7a966edcf63f41b65e78e886a Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sun, 20 Sep 2026 14:09:08 -0700 Subject: [PATCH 23/31] fix(rust): normalize numeric entity IDs like TypeScript --- rust/README.md | 17 +++++--- rust/src/lib.rs | 2 +- rust/src/use_case.rs | 61 ++++++++++++++++++++++++++--- rust/tests/key_spec.rs | 88 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 155 insertions(+), 13 deletions(-) create mode 100644 rust/tests/key_spec.rs diff --git a/rust/README.md b/rust/README.md index ad4774ba..9b1dc1c7 100644 --- a/rust/README.md +++ b/rust/README.md @@ -15,9 +15,10 @@ possible input or schedule. ## Use -The crate requires Rust 1.85 or later; CI pins 1.98.1 through -`rust/rust-toolchain.toml`, which rustup honors when cargo runs inside -`rust/`. Applications own their Redis connection and its timeout, retry and +The core crate requires Rust 1.85 or later; the `redis` feature requires Rust +1.88 with the currently locked Redis dependency. CI pins 1.98.1 through +`rust/rust-toolchain.toml`, which rustup honors when cargo runs inside `rust/`. +Applications own their Redis connection and its timeout, retry and resource budgets. The default runtime is the tokio runtime that is current while the cache is built. @@ -66,9 +67,13 @@ failure. A source deadline returns `Error::FallbackTimeout` and does not cancel the source. Dropping the future returned by `get` never cancels the execution: sources, publications and other callers keep their contracts. -`KeySpec` takes the entity id and ordered secondary dimensions; -`normalize_args` spells scalars the JavaScript way and orders names by UTF-16 -code units so the same identity produces the same Redis key in every language. +`KeySpec::new` accepts strings, integers and floats, including shared references +such as `&u64`. Its `IntoKeyId` conversion preserves string IDs and exact decimal +integers; floats use JavaScript number spelling, including negative zero and +exponents (`f32` is promoted to `f64`). For custom displayable IDs, pass +`id.to_string()` or implement `IntoKeyId`. `normalize_args` applies the shared +scalar spelling to secondary dimensions and orders names by UTF-16 code units +so the same identity produces the same Redis key in every language. Use the same namespace, key dimensions, codecs and policy across languages when sharing entries. diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 8635f8e5..7afbd9e4 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -74,4 +74,4 @@ pub use runtime::Runtime; #[cfg(feature = "tokio")] pub use runtime::TokioRuntime; pub use scope::Scope; -pub use use_case::{KeySpec, UseCase, UseCaseBuilder}; +pub use use_case::{IntoKeyId, KeySpec, UseCase, UseCaseBuilder}; diff --git a/rust/src/use_case.rs b/rust/src/use_case.rs index f4d59624..028cd719 100644 --- a/rust/src/use_case.rs +++ b/rust/src/use_case.rs @@ -11,7 +11,7 @@ use serde::Serialize; use crate::codec::{Codec, JsonCodec}; use crate::engine::DialCache; use crate::error::{BoxError, ConfigError, Error}; -use crate::identity::{normalize_args, ArgValue, Identity}; +use crate::identity::{js_number_to_string, normalize_args, ArgValue, Identity}; use crate::operation::{ downcast_value, erase_load, Comparator, ErasedOperation, Operation, Preview, RecoveryPredicate, SourceBudget, @@ -19,6 +19,54 @@ use crate::operation::{ use crate::policy::Policy; use crate::scope::Scope; +/// Converts an entity identifier to the text used in a cache key. +/// +/// Strings are preserved, integers use exact decimal notation, and floats use +/// JavaScript `String(number)` spelling (`f32` is promoted to `f64`). Shared +/// references to these types are also accepted. For other displayable IDs, +/// pass `id.to_string()` or implement this trait with the desired spelling. +pub trait IntoKeyId { + /// Consume the identifier and return its canonical text. + fn into_key_id(self) -> String; +} + +impl IntoKeyId for String { + fn into_key_id(self) -> String { + self + } +} + +impl IntoKeyId for &str { + fn into_key_id(self) -> String { + self.to_owned() + } +} + +impl IntoKeyId for &T { + fn into_key_id(self) -> String { + self.clone().into_key_id() + } +} + +macro_rules! integer_key_id { + ($($t:ty),*) => { $(impl IntoKeyId for $t { + fn into_key_id(self) -> String { self.to_string() } + })* }; +} +integer_key_id!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); + +impl IntoKeyId for f64 { + fn into_key_id(self) -> String { + js_number_to_string(self) + } +} + +impl IntoKeyId for f32 { + fn into_key_id(self) -> String { + js_number_to_string(f64::from(self)) + } +} + /// The key of one call: the entity id and any secondary dimensions. #[derive(Debug, Clone, PartialEq, Default)] pub struct KeySpec { @@ -30,10 +78,11 @@ pub struct KeySpec { } impl KeySpec { - /// A key with `id` and no secondary dimensions. - pub fn new(id: impl ToString) -> Self { + /// A key with `id` and no secondary dimensions. See [`IntoKeyId`] for + /// the shared numeric spelling and supported input types. + pub fn new(id: impl IntoKeyId) -> Self { KeySpec { - id: id.to_string(), + id: id.into_key_id(), args: Vec::new(), } } @@ -58,10 +107,10 @@ impl From<&str> for KeySpec { } } -macro_rules! key_from_integer { +macro_rules! key_from_number { ($($t:ty),*) => { $(impl From<$t> for KeySpec { fn from(id: $t) -> Self { KeySpec::new(id) } })* }; } -key_from_integer!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); +key_from_number!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize, f64, f32); impl From<&str> for ArgValue { fn from(value: &str) -> Self { diff --git a/rust/tests/key_spec.rs b/rust/tests/key_spec.rs new file mode 100644 index 00000000..db4db528 --- /dev/null +++ b/rust/tests/key_spec.rs @@ -0,0 +1,88 @@ +//! Native entity-ID conversion at the registered-use-case API boundary. + +// Borrowed input support is part of the public contract exercised below. +#![allow(clippy::needless_borrows_for_generic_args)] + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{DialCache, IntoKeyId, KeySpec, Policy}; + +#[test] +fn floating_ids_use_javascript_spelling_for_owned_and_borrowed_inputs() { + for (value, expected) in [ + (-0.0, "0"), + (1e-7, "1e-7"), + (1e21, "1e+21"), + (f64::from_bits(0x430c6bf526340002), "1000000000000000.2"), + (f64::from_bits(0xc30c6bf526340002), "-1000000000000000.2"), + (f64::INFINITY, "Infinity"), + (f64::NAN, "NaN"), + ] { + assert_eq!(KeySpec::new(value).id, expected); + assert_eq!(KeySpec::new(&value).id, expected); + assert_eq!(KeySpec::from(value).id, expected); + } +} + +#[test] +fn single_precision_ids_are_promoted_to_javascript_numbers() { + for (value, expected) in [(-0.0_f32, "0"), (0.1_f32, "0.10000000149011612")] { + assert_eq!(KeySpec::new(value).id, expected); + assert_eq!(KeySpec::new(&value).id, expected); + assert_eq!(KeySpec::from(value).id, expected); + } +} + +#[test] +fn string_and_integer_ids_keep_their_text_and_support_references() { + let text = String::from("001e+21"); + let borrowed_text = text.as_str(); + assert_eq!(KeySpec::new(text.as_str()).id, text); + assert_eq!(KeySpec::new(&borrowed_text).id, text); + assert_eq!(KeySpec::new(&text).id, text); + assert_eq!(KeySpec::new(text.clone()).id, text); + assert_eq!(KeySpec::from(text.clone()).id, text); + assert_eq!(KeySpec::new(&42_u64).id, "42"); + assert_eq!(KeySpec::new(&i128::MIN).id, i128::MIN.to_string()); + assert_eq!(KeySpec::new(u128::MAX).id, u128::MAX.to_string()); + assert_eq!(KeySpec::from(u128::MAX).id, u128::MAX.to_string()); + + // A custom Display type retains the explicit text escape hatch. + assert_eq!( + KeySpec::new(std::net::Ipv4Addr::LOCALHOST.to_string()).id, + "127.0.0.1" + ); + assert_eq!(42_u64.into_key_id(), "42"); +} + +#[test] +fn registered_float_ids_share_the_javascript_zero_identity() { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .build() + .unwrap(); + let sources = Arc::new(AtomicUsize::new(0)); + let calls = sources.clone(); + let lookup = cache + .use_case::("thing", "FloatId") + .policy(Policy::default().local_ttl_sec(60)) + .key(|id: &f64| KeySpec::new(id)) + .source(move |_, _| { + let value = calls.fetch_add(1, Ordering::SeqCst) + 1; + async move { Ok(value) } + }) + .register() + .unwrap(); + let (negative, positive) = executor.block_on(async move { + let request = cache.enable_guard(); + let negative = lookup.get(request.scope(), -0.0).await.unwrap(); + let positive = lookup.get(request.scope(), 0.0).await.unwrap(); + (negative, positive) + }); + assert_eq!((*negative, *positive), (1, 1)); + assert_eq!(sources.load(Ordering::SeqCst), 1); +} From 82e5ffb285b6aba5c235b3347460386d2d4668ce Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Sun, 20 Sep 2026 14:17:51 -0700 Subject: [PATCH 24/31] fix(rust): align invalidation IDs and complete native API ergonomics --- rust/README.md | 21 ++++--- rust/src/engine.rs | 6 +- rust/src/operation.rs | 15 ++++- rust/src/use_case.rs | 48 ++++++++++----- rust/tests/key_args.rs | 118 ++++++++++++++++++++++++++++++++++++ rust/tests/key_spec.rs | 101 +++++++++++++++++++++++++++++- rust/tests/operation_api.rs | 47 ++++++++++++++ 7 files changed, 328 insertions(+), 28 deletions(-) create mode 100644 rust/tests/key_args.rs create mode 100644 rust/tests/operation_api.rs diff --git a/rust/README.md b/rust/README.md index 9b1dc1c7..ad38131b 100644 --- a/rust/README.md +++ b/rust/README.md @@ -56,7 +56,8 @@ work that runs on behalf of no request. `UseCase` handle; `get_or_load` runs one inline `Operation` without registration. Both snapshot the static policy and the source budget before any asynchronous work. Values come back as `Arc`: shared by reference, treat -them as immutable. Use case `watermark` is reserved. `coalescing_state` +them as immutable. Both `UseCase` and `Operation` can be cloned +without requiring `T: Clone`. Use case `watermark` is reserved. `coalescing_state` reports actual process leaders, followers and the oldest leader age. Sources are `Fn(Scope, Args) -> Future>`. They may @@ -67,13 +68,17 @@ failure. A source deadline returns `Error::FallbackTimeout` and does not cancel the source. Dropping the future returned by `get` never cancels the execution: sources, publications and other callers keep their contracts. -`KeySpec::new` accepts strings, integers and floats, including shared references -such as `&u64`. Its `IntoKeyId` conversion preserves string IDs and exact decimal -integers; floats use JavaScript number spelling, including negative zero and -exponents (`f32` is promoted to `f64`). For custom displayable IDs, pass -`id.to_string()` or implement `IntoKeyId`. `normalize_args` applies the shared -scalar spelling to secondary dimensions and orders names by UTF-16 code units -so the same identity produces the same Redis key in every language. +`KeySpec::new` and `DialCache::invalidate` accept strings, integers and floats, +including shared references such as `&u64`. Their `IntoKeyId` conversion +preserves string IDs and exact decimal integers; floats use JavaScript number +spelling, including negative zero and exponents (`f32` is promoted to `f64`). +For custom displayable IDs, pass +`id.to_string()` or implement `IntoKeyId`. `KeySpec::arg` also accepts all +primitive integer and float types, preserving exact integer text and promoting +`f32` to `f64`, as well as borrowed inputs such as `&String` and `&u64`. +`normalize_args` applies the shared scalar spelling to secondary dimensions and +orders names by UTF-16 code units so the same identity produces the same Redis +key in every language. Use the same namespace, key dimensions, codecs and policy across languages when sharing entries. diff --git a/rust/src/engine.rs b/rust/src/engine.rs index 0e6d56e2..689d776e 100644 --- a/rust/src/engine.rs +++ b/rust/src/engine.rs @@ -27,6 +27,7 @@ use crate::remote::{InvalidateRequest, Remote}; use crate::runtime::Runtime; use crate::scope::{Owner, Scope}; use crate::shadow::ShadowFlight; +use crate::use_case::IntoKeyId; /// Resolves a sparse runtime policy overlay once per enabled call. pub type PolicyProvider = Arc< @@ -582,13 +583,14 @@ impl DialCache { /// Advance the invalidation watermark of every tracked variant of one /// entity, in this instance's namespace. Call it after the source /// mutation commits. Requires a remote adapter; failures are returned. + /// IDs use the same [`IntoKeyId`] conversion as [`crate::KeySpec::new`]. pub async fn invalidate( &self, key_type: &str, - id: &str, + id: impl IntoKeyId, future_buffer_ms: u64, ) -> Result<(), Error> { - let identity = Identity::new(key_type, id, WATERMARK_USE_CASE) + let identity = Identity::new(key_type, id.into_key_id(), WATERMARK_USE_CASE) .tracked(true) .namespace(self.core.namespace.to_string()); self.invalidate_identity(identity, future_buffer_ms).await diff --git a/rust/src/operation.rs b/rust/src/operation.rs index 20b62929..a0cc732d 100644 --- a/rust/src/operation.rs +++ b/rust/src/operation.rs @@ -51,7 +51,6 @@ pub type Preview = Arc Option + Send + Sync>; /// Registered use cases build this for every call; use it directly with /// [`DialCache::get_or_load`](crate::DialCache::get_or_load) when a stable use /// case is declared at the call site instead of being registered. -#[derive(Clone)] pub struct Operation { /// The logical identity; an empty namespace inherits the instance's. pub identity: Identity, @@ -70,6 +69,20 @@ pub struct Operation { pub preview: Option>, } +impl Clone for Operation { + fn clone(&self) -> Self { + Self { + identity: self.identity.clone(), + policy: self.policy.clone(), + budget: self.budget, + codec: self.codec.clone(), + comparator: self.comparator.clone(), + should_recover: self.should_recover.clone(), + preview: self.preview.clone(), + } + } +} + impl std::fmt::Debug for Operation { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("Operation") diff --git a/rust/src/use_case.rs b/rust/src/use_case.rs index 028cd719..dd42237b 100644 --- a/rust/src/use_case.rs +++ b/rust/src/use_case.rs @@ -88,7 +88,10 @@ impl KeySpec { } /// Add a secondary key dimension. Absent values are omitted; names are - /// sorted by UTF-16 code units when the key is built. + /// sorted by UTF-16 code units when the key is built. Primitive integers + /// preserve their exact decimal value; floats use JavaScript number + /// spelling, with `f32` promoted to `f64`. Shared references to supported + /// inputs are also accepted. pub fn arg(mut self, name: impl Into, value: impl Into) -> Self { self.args.push((name.into(), value.into())); self @@ -124,29 +127,36 @@ impl From for ArgValue { } } -impl From for ArgValue { - fn from(value: bool) -> Self { - ArgValue::Bool(value) +impl> From<&T> for ArgValue { + fn from(value: &T) -> Self { + value.clone().into() } } -impl From for ArgValue { - fn from(value: i64) -> Self { - ArgValue::Int(value) +impl From for ArgValue { + fn from(value: bool) -> Self { + ArgValue::Bool(value) } } -impl From for ArgValue { - fn from(value: i32) -> Self { - ArgValue::Int(value as i64) - } +macro_rules! integer_arg_value { + ($($t:ty),*) => { $(impl From<$t> for ArgValue { + fn from(value: $t) -> Self { ArgValue::Int(i64::from(value)) } + })* }; } - -impl From for ArgValue { - fn from(value: u32) -> Self { - ArgValue::Int(value as i64) - } +integer_arg_value!(i8, i16, i32, i64, u8, u16, u32); + +macro_rules! wide_integer_arg_value { + ($($t:ty),*) => { $(impl From<$t> for ArgValue { + fn from(value: $t) -> Self { + match i64::try_from(value) { + Ok(value) => ArgValue::Int(value), + Err(_) => ArgValue::BigInt(value.to_string()), + } + } + })* }; } +wide_integer_arg_value!(i128, isize, u64, u128, usize); impl From for ArgValue { fn from(value: f64) -> Self { @@ -154,6 +164,12 @@ impl From for ArgValue { } } +impl From for ArgValue { + fn from(value: f32) -> Self { + ArgValue::Number(f64::from(value)) + } +} + impl> From> for ArgValue { fn from(value: Option) -> Self { match value { diff --git a/rust/tests/key_args.rs b/rust/tests/key_args.rs new file mode 100644 index 00000000..0ea31d4c --- /dev/null +++ b/rust/tests/key_args.rs @@ -0,0 +1,118 @@ +//! Native scalar conversion at the secondary-key argument API boundary. + +// Borrowed inputs are part of the public contract exercised below. +#![allow(clippy::needless_borrows_for_generic_args)] + +use dialcache::{normalize_args, ArgValue, Identity, KeySpec}; + +fn normalized(value: impl Into) -> String { + let key = KeySpec::new("entity").arg("value", value); + normalize_args(key.args).unwrap().remove(0).1 +} + +#[test] +fn unsigned_and_wide_integer_arguments_preserve_every_decimal_digit() { + assert_eq!(normalized(9_007_199_254_740_993_u64), "9007199254740993"); + assert_eq!(normalized(u64::MAX), "18446744073709551615"); + assert_eq!( + normalized(i128::MIN), + "-170141183460469231731687303715884105728" + ); + assert_eq!( + normalized(i128::MAX), + "170141183460469231731687303715884105727" + ); + assert_eq!( + normalized(u128::MAX), + "340282366920938463463374607431768211455" + ); + assert_eq!(normalized(usize::MAX), usize::MAX.to_string()); + assert_eq!(normalized(isize::MIN), isize::MIN.to_string()); + assert_eq!(normalized(isize::MAX), isize::MAX.to_string()); +} + +#[test] +fn every_integer_primitive_accepts_zero_and_its_extremes() { + macro_rules! check { + ($($t:ty),*) => { $( + assert_eq!(normalized(0 as $t), "0", stringify!($t)); + assert_eq!(normalized(<$t>::MIN), <$t>::MIN.to_string(), stringify!($t)); + assert_eq!(normalized(<$t>::MAX), <$t>::MAX.to_string(), stringify!($t)); + )* }; + } + check!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); +} + +#[test] +fn arguments_keep_float_boolean_string_and_optional_scalar_semantics() { + for (value, expected) in [ + (0.1_f32, "0.10000000149011612"), + (-0.0_f32, "0"), + (f32::INFINITY, "Infinity"), + (f32::NEG_INFINITY, "-Infinity"), + (f32::NAN, "NaN"), + ] { + assert_eq!(normalized(value), expected); + } + assert_eq!( + normalized(f64::from_bits(0x430c6bf526340002)), + "1000000000000000.2" + ); + assert_eq!(normalized(false), "false"); + assert_eq!(normalized(true), "true"); + assert_eq!(normalized("001e+21"), "001e+21"); + assert_eq!(normalized(String::from("001e+21")), "001e+21"); + assert_eq!(normalized(ArgValue::Null), "null"); + + let key = KeySpec::new("entity") + .arg("present", Some(u128::MAX)) + .arg("omitted", None::) + .arg("absent", ArgValue::Absent); + assert_eq!( + normalize_args(key.args).unwrap(), + vec![( + "present".into(), + "340282366920938463463374607431768211455".into() + )] + ); +} + +#[test] +fn wide_arguments_form_distinct_keys_without_float_rounding() { + let key = |value: u128| { + let specification = KeySpec::new("id").arg("version", value); + Identity::new("entity", specification.id, "lookup") + .namespace("urn") + .args(normalize_args(specification.args).unwrap()) + .keys() + .unwrap() + .logical + }; + assert_eq!( + key(u128::MAX), + "urn:entity:id?version=340282366920938463463374607431768211455#lookup" + ); + assert_eq!( + key(u128::MAX - 1), + "urn:entity:id?version=340282366920938463463374607431768211454#lookup" + ); +} + +#[test] +fn borrowed_arguments_preserve_owned_scalar_spellings() { + let text = String::from("001e+21"); + let borrowed_text = text.as_str(); + assert_eq!(normalized(&text), text); + assert_eq!(normalized(&borrowed_text), text); + assert_eq!(normalized(&u64::MAX), "18446744073709551615"); + assert_eq!(normalized(&usize::MAX), usize::MAX.to_string()); + assert_eq!( + normalized(&u128::MAX), + "340282366920938463463374607431768211455" + ); + assert_eq!(normalized(&0.1_f32), "0.10000000149011612"); + assert_eq!(normalized(&false), "false"); + assert_eq!(normalized(&Some(u64::MAX)), "18446744073709551615"); + let key = KeySpec::new("entity").arg("omitted", &None::); + assert!(normalize_args(key.args).unwrap().is_empty()); +} diff --git a/rust/tests/key_spec.rs b/rust/tests/key_spec.rs index db4db528..6f8e678d 100644 --- a/rust/tests/key_spec.rs +++ b/rust/tests/key_spec.rs @@ -7,7 +7,12 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; -use dialcache::{DialCache, IntoKeyId, KeySpec, Policy}; +use dialcache::{ + BoxError, DialCache, IntoKeyId, InvalidateRequest, KeySpec, MissReason, Policy, ReadContext, + ReadRequest, ReadResult, Remote, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; #[test] fn floating_ids_use_javascript_spelling_for_owned_and_borrowed_inputs() { @@ -86,3 +91,97 @@ fn registered_float_ids_share_the_javascript_zero_identity() { assert_eq!((*negative, *positive), (1, 1)); assert_eq!(sources.load(Ordering::SeqCst), 1); } + +#[derive(Default)] +struct RecordingRemote { + reads: Mutex>, + invalidations: Mutex>, +} + +impl Remote for RecordingRemote { + fn read( + &self, + request: ReadRequest, + _: ReadContext, + ) -> BoxFuture<'_, Result> { + self.reads.lock().push(request); + Box::pin(async { Ok(ReadResult::miss(MissReason::ValueAbsent)) }) + } + + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(async { Ok(()) }) + } + + fn invalidate(&self, request: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + self.invalidations.lock().push(request); + Box::pin(async { Ok(()) }) + } +} + +fn assert_invalidation_matches_registered_id(id: T, escaped_id: &str) +where + T: IntoKeyId + Clone + Send + Sync + 'static, +{ + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let remote = Arc::new(RecordingRemote::default()); + let cache = DialCache::builder() + .namespace("numeric") + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote_arc(remote.clone()) + .build() + .unwrap(); + let lookup = cache + .use_case::("thing", "TrackedId") + .tracked(true) + .policy(Policy::default().remote_ttl_sec(60)) + .key(|id: &T| KeySpec::new(id)) + .source(|_, _| async { Ok(7) }) + .register() + .unwrap(); + executor.block_on(async move { + let request = cache.enable_guard(); + assert_eq!(*lookup.get(request.scope(), id.clone()).await.unwrap(), 7); + cache.invalidate("thing", &id, 17).await.unwrap(); + cache.invalidate("thing", id, 17).await.unwrap(); + }); + + let expected = format!("{{numeric:thing:{escaped_id}}}#watermark"); + let reads = remote.reads.lock(); + assert_eq!(reads.len(), 1); + assert_eq!(reads[0].watermark_key.as_deref(), Some(expected.as_str())); + let invalidations = remote.invalidations.lock(); + assert_eq!(invalidations.len(), 2); + for request in invalidations.iter() { + assert_eq!(request.watermark_key, expected); + assert_eq!(request.invalidated_at_ms, WALL_EPOCH_MS as u64); + assert_eq!(request.future_buffer_ms, 17); + } +} + +#[test] +fn numeric_invalidation_uses_the_registered_entity_watermark() { + for (id, escaped_id) in [ + (-0.0, "0"), + (1e-7, "1e-7"), + (1e21, "1e%2B21"), + (f64::from_bits(0x430c6bf526340002), "1000000000000000.2"), + (f64::INFINITY, "Infinity"), + (f64::NAN, "NaN"), + ] { + assert_invalidation_matches_registered_id(id, escaped_id); + } + assert_invalidation_matches_registered_id(0.1_f32, "0.10000000149011612"); + assert_invalidation_matches_registered_id(u64::MAX, "18446744073709551615"); + assert_invalidation_matches_registered_id( + i128::MIN, + "-170141183460469231731687303715884105728", + ); + assert_invalidation_matches_registered_id(u128::MAX, "340282366920938463463374607431768211455"); +} + +#[test] +fn string_invalidation_preserves_explicit_entity_text() { + assert_invalidation_matches_registered_id("001e+21", "001e%2B21"); + assert_invalidation_matches_registered_id(String::from("-0"), "-0"); +} diff --git a/rust/tests/operation_api.rs b/rust/tests/operation_api.rs new file mode 100644 index 00000000..17e619ed --- /dev/null +++ b/rust/tests/operation_api.rs @@ -0,0 +1,47 @@ +//! Inline operations remain reusable when the cached value cannot be cloned. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; + +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{DialCache, Identity, Operation, Policy}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, PartialEq, Serialize, Deserialize)] +struct NonCloneValue { + value: usize, +} + +#[test] +fn cloned_operation_reuses_non_clone_cached_values() { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .build() + .unwrap(); + let sources = Arc::new(AtomicUsize::new(0)); + let calls = sources.clone(); + let operation = Operation::::new(Identity::new("thing", "one", "NonClone")) + .policy(Policy::default().local_ttl_sec(60)); + let (first, second) = executor.block_on(async move { + let request = cache.enable_guard(); + let load = move |_| { + let value = calls.fetch_add(1, Ordering::SeqCst) + 1; + async move { Ok(NonCloneValue { value }) } + }; + let first = cache + .get_or_load(request.scope(), operation.clone(), load.clone()) + .await + .unwrap(); + let second = cache + .get_or_load(request.scope(), operation, load) + .await + .unwrap(); + (first, second) + }); + + assert_eq!(first.value, 1); + assert!(Arc::ptr_eq(&first, &second)); + assert_eq!(sources.load(Ordering::SeqCst), 1); +} From 3d322852de70870a760fe7a4bd7f4529f652164d Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Mon, 21 Sep 2026 16:05:57 -0700 Subject: [PATCH 25/31] fix(rust): adopt current replay contracts and unify identity ergonomics --- formal/SEMANTIC-COVERAGE.md | 6 +- formal/go-parity.json | 4 +- formal/measure-rust-semantics.mjs | 52 ++++++--- formal/mutation-reports.mjs | 7 +- formal/profiles.json | 4 +- formal/source-audit.json | 34 +++--- rust/Cargo.toml | 8 ++ rust/README.md | 66 ++++++++---- rust/examples/basic.rs | 50 +++++++++ rust/examples/redis.rs | 45 ++++++++ rust/src/engine.rs | 3 +- rust/src/identity.rs | 73 ++++++++++++- rust/src/lib.rs | 4 +- rust/src/testing.rs | 33 +++++- rust/src/use_case.rs | 50 +-------- rust/tests/conformance.rs | 43 ++++---- rust/tests/formal/driver.rs | 105 ++++++++++++++---- rust/tests/formal/inventory.rs | 70 ++++-------- rust/tests/formal/transport.rs | 40 ++++++- rust/tests/formal/witness.rs | 49 ++++++++- rust/tests/harness_infra.rs | 131 ++++++++++++++++++++++- rust/tests/key_spec.rs | 69 ++++++++++-- rust/tests/settlement_control.rs | 56 ++++++++-- test/formal-mutation-shards.test.ts | 39 +++++-- test/formal-rust-replay.test.ts | 16 ++- test/formal-rust-semantic-runner.test.ts | 30 +++++- 26 files changed, 833 insertions(+), 254 deletions(-) create mode 100644 rust/examples/basic.rs create mode 100644 rust/examples/redis.rs diff --git a/formal/SEMANTIC-COVERAGE.md b/formal/SEMANTIC-COVERAGE.md index 2c2cbf20..0d8a1cbc 100644 --- a/formal/SEMANTIC-COVERAGE.md +++ b/formal/SEMANTIC-COVERAGE.md @@ -183,8 +183,10 @@ runner requires complete reports and assertion evidence; an infrastructure failure is a failed measurement. Rust results establish detection for this catalog only; the TypeScript/Go model-to-mutant boundary mappings in `mutations.json` do not confer Rust coverage. Each mutant builds in release -mode in an isolated crate copy. Concurrent local shards need separate -`DIALCACHE_RUST_TARGET_DIR` build directories to avoid sharing mutant binaries. +mode in an isolated crate copy. Cargo build directories are isolated by shard +selection automatically; partial runs use a separate directory. A direct runner +invocation can override the directory with `DIALCACHE_RUST_TARGET_DIR`, but +concurrent shards must use distinct override paths to avoid sharing mutant binaries. Hosted runs shard each lane with `MUTATION_SHARD=/`, matching the workflow matrix; `test/formal-validation.test.ts` pins how many mutants a shard may hold within its timeout, so catalog growth fails the pull request until the matrix grows. A shard reruns the compile check, every unmodified baseline and the witness diff --git a/formal/go-parity.json b/formal/go-parity.json index 7645cd7a..6dc88cdb 100644 --- a/formal/go-parity.json +++ b/formal/go-parity.json @@ -10,10 +10,10 @@ "inputs": { "semanticCasesSha256": "faa768fbf06eddc03dc4f26570094b1df066fb075f898043b40c6ce9d8c952a1", "executionSha256": "3df06c53eb69f02fb6eafabd51c6cc7a306c6739e04ab3fa521b53389da559f2", - "sourceAuditSha256": "2b749b2b0a17cc7134108fe939fdb7d7792c19883c591379af4b66f2135cd699", + "sourceAuditSha256": "1634e1c839d88babf762b44aaf2db9e0ea514409b2689581327fdf4a00fd40d9", "featureCoverageSha256": "91723cc5398c8fff102d2beb199ebcab82af137e11f1cb9771b8d16d0304a95c", "coverageWitnessesSha256": "2b2d5d57daacba3ecd3637f5a917d58b0b63fcc88893e651cd602cdba1ef4244", - "profilesSha256": "f8ec93fe5aa23faa69edd0a632a645874a6190a48ed613d2c65f82045219f49c" + "profilesSha256": "2ebb3ba9c9b71334bcbbaa1e2d9c1d539664ed69e826942ff829a2c31fe9434b" }, "inventory": { "behavioralCases": 244, diff --git a/formal/measure-rust-semantics.mjs b/formal/measure-rust-semantics.mjs index 882befbe..8d8b3ab6 100644 --- a/formal/measure-rust-semantics.mjs +++ b/formal/measure-rust-semantics.mjs @@ -5,7 +5,8 @@ import { resolve, relative } from 'node:path'; import { tmpdir } from 'node:os'; import { fileURLToPath } from 'node:url'; import { causalPropertyAssertion } from './measure-go-semantics.mjs'; -import { fingerprintFiles, gateDetections, languages, partitionMutations, shardDirectory, shardFromArguments } from './mutation-reports.mjs'; +import { mutantCatalogPath, mutantsForPort, readMutantCatalog } from './execution.mjs'; +import { fingerprintFiles, gateDetections, languages, selectMutations, selectionDirectory, selectionFromArguments } from './mutation-reports.mjs'; // The Rust counterpart of measure-go-semantics.mjs: apply each catalogued // fault to an isolated copy of the crate, require it to compile, and run three @@ -100,20 +101,45 @@ function union(generated, fixed) { failed: generated.failed + fixed.failed, failingTests: [...generated.failingTests, ...fixed.failingTests], components: ['generated', 'fixed'] }; } +// This native catalog is a deliberately smaller set than the current shared +// TypeScript/Go catalog. Record the gap instead of claiming complete coverage +// of its model-challenge boundary recordings. +export function rustMutationScope(catalog, typescript) { + if (catalog.schemaVersion !== 1 || !Array.isArray(catalog.mutations) || catalog.mutations.length < 13) throw new Error('expected versioned Rust fault catalog with its 13 baseline faults'); + const ids = new Set(), counterparts = new Set(); + for (const mutation of catalog.mutations) { + if (!/^M\d+$/.test(mutation.id) || ids.has(mutation.id) || counterparts.has(mutation.typescriptMutation)) throw new Error('invalid/duplicate Rust mutation or counterpart ID'); + ids.add(mutation.id); counterparts.add(mutation.typescriptMutation); + const counterpart = typescript.find(item => item.id === mutation.typescriptMutation); + if (!counterpart || counterpart.case !== mutation.case) throw new Error(`invalid TypeScript counterpart ${mutation.id}`); + } + return { catalog: 'rust-native', modelBoundaryEvidence: false, + mappedMutations: [...counterparts], + unmappedSharedMutations: typescript.filter(item => !counterparts.has(item.id)).map(item => item.id) }; +} + +// Concurrent shards must never execute a sibling shard's mutated binaries. +// Keep the Cargo build cache separate by the same selection used for reports. +export function rustTargetDirectory(directory, selection) { + return selectionDirectory(resolve(directory, 'rust/target/semantic'), selection); +} + export function measureRustSemantics({ shard = { index: 1, count: 1 }, only } = {}) { const language = languages.rust; const reportRoot = resolve(root, language.output); - const output = shardDirectory(reportRoot, shard); + const selection = { shard, only }; + const output = selectionDirectory(reportRoot, selection); const started = Date.now(); mkdirSync(reportRoot, { recursive: true }); const report = { schemaVersion: 1, complete: false, ...(shard.count > 1 ? { shard: { index: shard.index, count: shard.count } } : {}), startedAt: new Date(started).toISOString(), baselines: {}, mutations: [] }; const save = () => { report.elapsedSeconds = Math.round((Date.now() - started) / 1000); writeFileSync(resolve(output, 'report.json'), JSON.stringify(report, null, 2) + '\n'); }; if (output !== reportRoot) { - writeFileSync(resolve(reportRoot, 'report.json'), JSON.stringify({ schemaVersion: 1, complete: false, startedAt: report.startedAt }, null, 2) + '\n'); + if (only === undefined) writeFileSync(resolve(reportRoot, 'report.json'), JSON.stringify({ schemaVersion: 1, complete: false, startedAt: report.startedAt }, null, 2) + '\n'); rmSync(output, { recursive: true, force: true }); mkdirSync(output, { recursive: true }); } - save(); rmSync(resolve(reportRoot, 'report.md'), { force: true }); + save(); + if (only === undefined) rmSync(resolve(reportRoot, 'report.md'), { force: true }); const workspace = mkdtempSync(resolve(tmpdir(), 'dialcache-rust-semantic-')); const cargo = process.env.CARGO_BIN ?? 'cargo'; // Bounds a hung mutant, not a slow runner: one release build of the crate @@ -131,14 +157,13 @@ export function measureRustSemantics({ shard = { index: 1, count: 1 }, only } = const crate = resolve(workspace, 'rust'); const catalogPath = resolve(workspace, language.catalog); const catalog = json(catalogPath); - const typescript = json(resolve(workspace, 'formal/semantic-mutations.json')); - if (catalog.schemaVersion !== 1 || !Array.isArray(catalog.mutations) || catalog.mutations.length < 13) throw new Error('expected versioned Rust fault catalog with all 13 TypeScript counterparts'); + const typescript = mutantsForPort(readMutantCatalog(path => readFileSync(resolve(workspace, path), 'utf8')), 'typescript'); + report.scope = rustMutationScope(catalog, typescript); + report.sharedCatalogSha256 = hash(readFileSync(resolve(workspace, mutantCatalogPath))); const originals = new Map(), ids = new Set(); for (const mutation of catalog.mutations) { if (!/^M\d+$/.test(mutation.id) || ids.has(mutation.id)) throw new Error('invalid/duplicate mutation ID'); ids.add(mutation.id); - const counterpart = typescript.mutations.find(item => item.id === mutation.typescriptMutation); - if (!counterpart || counterpart.case !== mutation.case) throw new Error(`invalid TypeScript counterpart ${mutation.id}`); if (!Array.isArray(mutation.edits) || !mutation.edits.length || !mutation.requiredDetections?.every(name => ['ordinary', 'generated', 'fixed', 'portable'].includes(name))) throw new Error(`invalid mutation ${mutation.id}`); for (const edit of mutation.edits) { if (!/^rust\/src\/[\w/-]+\.rs$/.test(edit.path) || !edit.before || edit.before === edit.after) throw new Error(`invalid production edit ${mutation.id}`); @@ -147,9 +172,7 @@ export function measureRustSemantics({ shard = { index: 1, count: 1 }, only } = originals.set(edit.path, original); } } - for (const counterpart of typescript.mutations) if (!catalog.mutations.some(m => m.typescriptMutation === counterpart.id)) throw new Error(`missing TypeScript counterpart ${counterpart.id}`); - const selected = only ? catalog.mutations.filter(m => only.includes(m.id)) : partitionMutations(catalog.mutations, shard); - if (only && selected.length !== only.length) throw new Error(`--only names unknown mutations: ${only.filter(id => !selected.some(m => m.id === id)).join(', ')}`); + const selected = selectMutations(catalog.mutations, selection); if (shard.count > 1) report.shard = { index: shard.index, count: shard.count, mutationIds: selected.map(m => m.id) }; // Ordinary tests: the library's unit tests and every integration binary // that is not harness infrastructure, discovered from the crate so a new @@ -174,7 +197,7 @@ export function measureRustSemantics({ shard = { index: 1, count: 1 }, only } = // Dependencies build once and are shared by every mutant and run through // the checkout's cache directory; the crate copy itself lives at another // path, so cargo rebuilds exactly the crate and its tests per mutant. - env.CARGO_TARGET_DIR = process.env.DIALCACHE_RUST_TARGET_DIR ?? resolve(root, 'rust/target/semantic'); + env.CARGO_TARGET_DIR = process.env.DIALCACHE_RUST_TARGET_DIR ?? rustTargetDirectory(root, selection); env.CARGO_TERM_COLOR = 'never'; report.revision = spawnSync('git', ['rev-parse', 'HEAD'], { cwd: root, encoding: 'utf8' }).stdout.trim(); report.cargo = spawnSync(cargo, ['--version'], { cwd: crate, encoding: 'utf8' }).stdout?.trim(); @@ -265,9 +288,6 @@ export function measureRustSemantics({ shard = { index: 1, count: 1 }, only } = if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { try { - const only = process.argv.slice(2).filter(argument => argument.startsWith('--only=')).flatMap(argument => argument.slice('--only='.length).split(',').filter(Boolean)); - const shard = shardFromArguments(process.argv.slice(2).filter(argument => !argument.startsWith('--only='))); - if (only.length && shard.count > 1) throw new Error('--only cannot be combined with --shard'); - measureRustSemantics({ shard, only: only.length ? only : undefined }); + measureRustSemantics(selectionFromArguments(process.argv.slice(2))); } catch (error) { console.error(error); process.exitCode = 1; } } diff --git a/formal/mutation-reports.mjs b/formal/mutation-reports.mjs index de4e4eb3..b9e09add 100644 --- a/formal/mutation-reports.mjs +++ b/formal/mutation-reports.mjs @@ -270,6 +270,8 @@ function goMarkdown(report, directory = root) { function rustMarkdown(report) { return ['# Rust semantic mutation measurement', '', `Completed in ${report.elapsedSeconds}s. Counts measure this named fault catalog and exact corpus, not universal equivalence.`, '', ...shardsMarkdown(report), + 'Scope: Rust native mutation catalog only. Shared TypeScript/Go model-challenge boundary coverage is not measured by this report.', '', + ...(report.scope ? [`Shared catalog mutants without a Rust binding: ${report.scope.unmappedSharedMutations.join(', ') || 'none'}.`, ''] : []), '| Mutation | Contract case | Ordinary | Quint generated | Fixed supplement | Full portable |', '| --- | --- | --- | --- | --- | --- |', ...report.mutations.map(m => `| ${m.id} | ${m.case} | ${m.cohorts.ordinary.state} | ${m.cohorts.generated.state} | ${m.cohorts.fixed.state} | ${m.cohorts.portable.state} |`), '', 'Ordinary is the crate\'s unit and native tests; generated is the conformance harness over the complete corpus and witness evidence; fixed is the harness over the fixed scenarios and checked-in vectors. Full JSON records snapshot/corpus/witness fingerprints, cohort selections, actual passing/failing counts and assertion diagnostics. Compilation errors, crashes, timeouts, missing reports and incomplete runs cannot count as detections.', ''].join('\n'); @@ -293,7 +295,7 @@ export const languages = { detection: mutations => goDetection(mutations), markdown: goMarkdown, recordsRegressions: true }, // The crate's unit tests read go/redis_adapter.go (script byte equality), so // the Go source is an input of the ordinary cohort as well. - rust: { name: 'Rust', output: '.formal-traces/rust-semantic', catalog: 'formal/rust-mutations.json', inputs: ['formal', 'rust', 'test', 'src', 'go'], exclude: ['rust/target'], + rust: { name: 'Rust', boundaryEvidence: false, output: '.formal-traces/rust-semantic', catalog: 'formal/rust-mutations.json', inputs: ['formal', 'rust', 'test', 'src', 'go'], exclude: ['rust/target'], detection: mutations => goDetection(mutations), markdown: rustMarkdown, recordsRegressions: true }, }; @@ -334,7 +336,8 @@ export function gateDetections(language, report, entries, { directory = root, su // Recompute verdicts from the current declarations and completed recordings. // An omitted mapping, a stale pin or a claimed confirmation without a clean // baseline must fail just as a measured non-detection does. - const regressions = requiredDetectionRegressions(entries, report.mutations, currentBoundaries(report, entries, directory)); + const boundaries = language.boundaryEvidence === false ? [] : currentBoundaries(report, entries, directory); + const regressions = requiredDetectionRegressions(entries, report.mutations, boundaries); if (summarize) report.detection = language.detection(report.mutations, directory); if (language.recordsRegressions) report.requiredDetectionRegressions = regressions; if (regressions.length) throw new Error(`Lost required detections: ${regressions.join(', ')}`); diff --git a/formal/profiles.json b/formal/profiles.json index efb7d46d..23bc2b3e 100644 --- a/formal/profiles.json +++ b/formal/profiles.json @@ -364,7 +364,9 @@ "runtime-boundaries", "shadow-layers", "local-clock", - "source-budgets" + "source-budgets", + "dark-layers", + "shadow-read-deadlines" ], "limits": "Same generated histories and portable scenarios as TypeScript and Go and all protocol vectors, replayed by the Rust conformance harness against the shared witness evidence. Claims require current corpus and witness fingerprints; finite evidence does not prove every possible schedule. Native Redis integration is not yet part of the claim." } diff --git a/formal/source-audit.json b/formal/source-audit.json index 1b553d4b..18786505 100644 --- a/formal/source-audit.json +++ b/formal/source-audit.json @@ -3,7 +3,7 @@ "sources": [ { "path": "README.md", - "sha256": "17cd061cde252e466ba94fced8615d3bc7d2a6b88cced14fe62c3572dc86c173", + "sha256": "73fd3028635e25c046ff3af486f58ea18b2754038b60c58e12c7d9cc2624bfa5", "entries": [ {"line":1,"title":"DialCache","contracts":["B01"]}, {"line":24,"title":"Install","contracts":["B01"]}, @@ -1288,7 +1288,7 @@ }, { "path": "formal/PORTING.md", - "sha256": "9eed42f975c129b1b001b7189c3564dff79f25ae9a38f299853209a8f091e300", + "sha256": "e25af32085f0e845e98ec32c1f988accc451f0913050b483456069a086d2430f", "entries": [ {"line":1,"title":"Implementing another DialCache port"}, {"line":14,"title":"What a port supplies"}, @@ -1297,9 +1297,9 @@ {"line":196,"title":"Observation contract"}, {"line":294,"title":"Generated fixtures and fast local tests"}, {"line":329,"title":"Complete corpus and reusable acceptance checks"}, - {"line":437,"title":"Witness evidence"}, - {"line":527,"title":"Current limitations for a third port"}, - {"line":549,"title":"New-port acceptance"} + {"line":442,"title":"Witness evidence"}, + {"line":532,"title":"Current limitations for a further port"}, + {"line":554,"title":"New-port acceptance"} ], "review": { "kind": "tooling-guide", @@ -1347,15 +1347,15 @@ }, { "path": "formal/README.md", - "sha256": "acb61cc27f29cb9b3f8996fef1d72ae301ebff65ff0793515c62778cc657a6a0", + "sha256": "da2c12a96c6a63bf11098dd28a0512e334adf5362834828b35e846bc1dbd1a52", "entries": [ {"line":1,"title":"Executable DialCache specification"}, {"line":10,"title":"Start with your task"}, {"line":26,"title":"How the specification connects to code"}, {"line":46,"title":"Models and composition profiles"}, {"line":99,"title":"Generating and replaying behavior"}, - {"line":193,"title":"Shared verification and replay rules"}, - {"line":209,"title":"Evidence and scope"} + {"line":194,"title":"Shared verification and replay rules"}, + {"line":210,"title":"Evidence and scope"} ], "review": { "kind": "tooling-guide", @@ -1364,7 +1364,7 @@ }, { "path": "formal/SEMANTIC-COVERAGE.md", - "sha256": "788ccd40ccc5f3e3462ea25fab7ba577d84da0f52a0f6d5a1d1a63b7a7a18a7f", + "sha256": "cdaf28072d154b3503be5939ef7828073832ff8044546019fa4c35b988781810", "entries": [ {"line":1,"title":"Measuring semantic coverage"}, {"line":5,"title":"Evidence inventory"}, @@ -1373,7 +1373,7 @@ {"line":139,"title":"Behavioral mutation comparison"}, {"line":153,"title":"Recorded measurements"}, {"line":159,"title":"Reproduction and CI"}, - {"line":219,"title":"Model properties and cross-language execution"} + {"line":234,"title":"Model properties and cross-language execution"} ], "review": { "kind": "coverage-guide", @@ -1503,15 +1503,15 @@ }, { "path": "formal/VALIDATION.md", - "sha256": "7747e7e1668b8b9557306cf7724182c0995b84b0044c3434501ae0a9e37d3f21", + "sha256": "2feaf2ec97a22b0d4da5eb8c79302cbb4e12416e86762be90b5ab2ff632ee952", "entries": [ {"line":1,"title":"Validation and evidence"}, {"line":7,"title":"Choosing a run"}, - {"line":76,"title":"Reading a completion report"}, - {"line":94,"title":"Mutation evidence"}, - {"line":205,"title":"Exploratory runs"}, - {"line":256,"title":"A retained sampling regression"}, - {"line":293,"title":"Historical results"} + {"line":77,"title":"Reading a completion report"}, + {"line":96,"title":"Mutation evidence"}, + {"line":210,"title":"Exploratory runs"}, + {"line":261,"title":"A retained sampling regression"}, + {"line":298,"title":"Historical results"} ], "review": { "kind": "tooling-guide", @@ -1535,7 +1535,7 @@ }, { "path": "go/README.md", - "sha256": "6d6a8f5ac238772a100ac606199889c3a043562e5c718fbb0af85a4cc84166ad", + "sha256": "6f5e6f576990718ae212092a42dc1a3d3f62d2d88e8999ed06d21ac9a0dde98c", "entries": [ {"line":1,"title":"DialCache for Go"}, {"line":19,"title":"Use"}, diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 9235c27e..fafe1a58 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -50,3 +50,11 @@ harness = false name = "redis_integration" path = "tests/redis_integration.rs" required-features = ["redis"] + +[[example]] +name = "basic" +required-features = ["tokio"] + +[[example]] +name = "redis" +required-features = ["redis", "tokio"] diff --git a/rust/README.md b/rust/README.md index ad38131b..12d002ad 100644 --- a/rust/README.md +++ b/rust/README.md @@ -24,24 +24,54 @@ while the cache is built. ```rust use std::sync::Arc; -use dialcache::{DialCache, KeySpec, Policy}; - -let cache = DialCache::builder() - .remote(dialcache::redis::RedisAdapter::new(connection)) // feature "redis" - .build()?; - -let display_name = cache - .use_case::("user", "displayName") - .policy(Policy::default().local_ttl_sec(1).remote_ttl_sec(60)) - .tracked(true) - .key(|id: &u64| KeySpec::new(id)) - .source(|_scope, id: u64| async move { Ok(load_display_name(id).await?) }) - .register()?; - -let request = cache.enable_guard(); -let name: Arc = display_name.get(request.scope(), 42).await?; +use dialcache::{BoxError, DialCache, KeySpec, Policy}; + +#[tokio::main] +async fn main() -> Result<(), BoxError> { + let cache = DialCache::builder().namespace("my-app").build()?; + let display_name = cache + .use_case::("user", "displayName") + .policy(Policy::default().request_local(true).local_ttl_sec(30)) + .key(|id: &u64| KeySpec::new(id)) + .source(|_scope, id| async move { + // Replace this with your database or API call. + Ok(format!("User {id}")) + }) + .register()?; + + let request = cache.enable_guard(); + let name: Arc = display_name.get(request.scope(), 42).await?; + println!("Hello, {name}!"); + Ok(()) +} ``` +A policy enables no cache layers by default. The example opts into request +caching and a 30-second process-local cache; register the use case once at +startup and create a scope for each request. `Arc` shares one cached value +without requiring `T: Clone`. + +Run the complete [basic example](./examples/basic.rs), which demonstrates a +structured value, an asynchronous source and reuse across two request scopes: + +```sh +cd rust +cargo run --example basic +``` + +The [Redis example](./examples/redis.rs) configures application-owned connection +and command timeouts, enables tracked Redis caching and demonstrates +invalidation. With a Redis server running: + +```sh +REDIS_URL=redis://127.0.0.1/ cargo run --features redis --example redis +``` + +The examples use the crate's existing dependencies. Applications also need +`tokio` with `macros`, `rt-multi-thread` and `time` enabled; structured JSON +values use `serde` with its `derive` feature. The Redis example additionally +needs the `redis` crate with `tokio-comp`, and DialCache's `redis` feature. + Caching is disabled by default. `DialCache::enable` (closure form) or `DialCache::enable_guard` (RAII form) opens the outermost enabled scope and hands out a `Scope`; pass it to every cached call made on behalf of that @@ -68,8 +98,8 @@ failure. A source deadline returns `Error::FallbackTimeout` and does not cancel the source. Dropping the future returned by `get` never cancels the execution: sources, publications and other callers keep their contracts. -`KeySpec::new` and `DialCache::invalidate` accept strings, integers and floats, -including shared references such as `&u64`. Their `IntoKeyId` conversion +`Identity::new`, `KeySpec::new` and `DialCache::invalidate` accept strings, +integers and floats, including shared references such as `&u64`. Their `IntoKeyId` conversion preserves string IDs and exact decimal integers; floats use JavaScript number spelling, including negative zero and exponents (`f32` is promoted to `f64`). For custom displayable IDs, pass diff --git a/rust/examples/basic.rs b/rust/examples/basic.rs new file mode 100644 index 00000000..0bc16f0a --- /dev/null +++ b/rust/examples/basic.rs @@ -0,0 +1,50 @@ +use std::sync::Arc; +use std::time::Duration; + +use dialcache::{BoxError, DialCache, KeySpec, Policy}; +use serde::{Deserialize, Serialize}; + +// Results need serialization and comparison for the default codec and shadow +// comparator. They do not need Clone: cached callers share an Arc. +#[derive(Debug, Serialize, Deserialize, PartialEq)] +struct User { + id: u64, + name: String, +} + +async fn load_user(id: u64) -> Result { + println!("Loading user {id} from the source"); + // Replace this with your database or API call. + tokio::time::sleep(Duration::from_millis(10)).await; + Ok(User { + id, + name: "Ada".to_owned(), + }) +} + +#[tokio::main] +async fn main() -> Result<(), BoxError> { + // Application startup: build the cache and register each use case once. + let cache = DialCache::builder().namespace("my-app").build()?; + let users = cache + .use_case::("user", "byId") + .policy(Policy::default().request_local(true).local_ttl_sec(30)) + .key(|id: &u64| KeySpec::new(id)) + .source(|_scope, id| load_user(id)) + .register()?; + + // Request handler: pass this scope to all cached calls in the request. + let request = cache.enable_guard(); + let user: Arc = users.get(request.scope(), 42).await?; + let again = users.get(request.scope(), 42).await?; + assert!(Arc::ptr_eq(&user, &again)); + println!("Hello, {}! Repeated calls share the same value.", user.name); + drop(request); // Closes the request cache; the returned Arc remains usable. + + // Another request can still use the 30-second process-local entry. + let next_request = cache.enable_guard(); + let next = users.get(next_request.scope(), 42).await?; + assert!(Arc::ptr_eq(&user, &next)); + println!("The next request reused the process-local entry."); + Ok(()) +} diff --git a/rust/examples/redis.rs b/rust/examples/redis.rs new file mode 100644 index 00000000..a34236bc --- /dev/null +++ b/rust/examples/redis.rs @@ -0,0 +1,45 @@ +use std::env; +use std::time::Duration; + +use dialcache::{BoxError, DialCache, KeySpec, Policy, RedisAdapter}; +use redis::AsyncConnectionConfig; + +#[tokio::main] +async fn main() -> Result<(), BoxError> { + let url = env::var("REDIS_URL").unwrap_or_else(|_| "redis://127.0.0.1/".to_owned()); + // Connection establishment, command deadlines and concurrency belong to + // the application. This connection does not automatically retry commands. + let config = AsyncConnectionConfig::new() + .set_connection_timeout(Some(Duration::from_secs(2))) + .set_response_timeout(Some(Duration::from_millis(250))) + .set_concurrency_limit(64); + let connection = redis::Client::open(url)? + .get_multiplexed_async_connection_with_config(&config) + .await?; + let cache = DialCache::builder() + .namespace("dialcache-example") + .remote(RedisAdapter::new(connection)) + .build()?; + let names = cache + .use_case::("user", "displayName") + .policy(Policy::default().remote_ttl_sec(300)) + .tracked(true) + .key(|id: &u64| KeySpec::new(id)) + .source(|_scope, id| async move { + println!("Loading user {id} from the source"); + // Replace this with your database or API call. + Ok(format!("User {id}")) + }) + .register()?; + + let request = cache.enable_guard(); + let name = names.get(request.scope(), 42).await?; + println!("Hello, {name}!"); + drop(request); + + // Invalidate the demo's tracked Redis entry. In an application, call this + // after successfully updating the source. Zero adds no future time buffer. + // Existing request/process-local entries retain their normal lifetimes. + cache.invalidate("user", 42_u64, 0).await?; + Ok(()) +} diff --git a/rust/src/engine.rs b/rust/src/engine.rs index 689d776e..e3defbb7 100644 --- a/rust/src/engine.rs +++ b/rust/src/engine.rs @@ -13,7 +13,7 @@ use parking_lot::Mutex; use crate::clock::Clock; use crate::error::{BoxError, ConfigError, Error}; use crate::flight::{start_pending, Flight, Settled, ValueResult}; -use crate::identity::Identity; +use crate::identity::{Identity, IntoKeyId}; use crate::limits::{ DEFAULT_LOCAL_CAPACITY, DEFAULT_REMOTE_READ_TIMEOUT_MS, DEFAULT_SHADOW_MAX_IN_FLIGHT, MAX_DEADLINE_MS, MAX_SAFE_INTEGER, MAX_SUPPORTED_DURATION_MS, WATERMARK_USE_CASE, @@ -27,7 +27,6 @@ use crate::remote::{InvalidateRequest, Remote}; use crate::runtime::Runtime; use crate::scope::{Owner, Scope}; use crate::shadow::ShadowFlight; -use crate::use_case::IntoKeyId; /// Resolves a sparse runtime policy overlay once per enabled call. pub type PolicyProvider = Arc< diff --git a/rust/src/identity.rs b/rust/src/identity.rs index 0afd3511..1a1c88ce 100644 --- a/rust/src/identity.rs +++ b/rust/src/identity.rs @@ -4,11 +4,78 @@ //! escaping, tracked entity hash tags, ordered argument pairs, the frame key //! suffix, and FNV-1a cohorts over UTF-16 code units. +use std::borrow::Cow; use std::cmp::Ordering; use std::fmt::Write as _; use crate::limits::FRAME_KEY_SUFFIX; +/// Converts an entity identifier to the text used in a cache key. +/// +/// Strings are preserved, integers use exact decimal notation, and floats use +/// JavaScript `String(number)` spelling (`f32` is promoted to `f64`). Shared +/// references to these types are also accepted. For other displayable IDs, +/// pass `id.to_string()` or implement this trait with the desired spelling. +pub trait IntoKeyId { + /// Consume the identifier and return its canonical text. + fn into_key_id(self) -> String; +} + +impl IntoKeyId for String { + fn into_key_id(self) -> String { + self + } +} + +impl IntoKeyId for &str { + fn into_key_id(self) -> String { + self.to_owned() + } +} + +impl IntoKeyId for Box { + fn into_key_id(self) -> String { + self.into_string() + } +} + +impl IntoKeyId for Cow<'_, str> { + fn into_key_id(self) -> String { + self.into_owned() + } +} + +impl IntoKeyId for char { + fn into_key_id(self) -> String { + self.to_string() + } +} + +impl IntoKeyId for &T { + fn into_key_id(self) -> String { + self.clone().into_key_id() + } +} + +macro_rules! integer_key_id { + ($($t:ty),*) => { $(impl IntoKeyId for $t { + fn into_key_id(self) -> String { self.to_string() } + })* }; +} +integer_key_id!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); + +impl IntoKeyId for f64 { + fn into_key_id(self) -> String { + js_number_to_string(self) + } +} + +impl IntoKeyId for f32 { + fn into_key_id(self) -> String { + js_number_to_string(f64::from(self)) + } +} + /// A normalized logical identity. Ordered arguments retain caller order; /// use [`normalize_args`] to build them from a host-language record. #[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] @@ -36,15 +103,17 @@ pub struct Identity { impl Identity { /// An untracked identity with an empty namespace and no arguments. + /// IDs use the same [`IntoKeyId`] conversion as [`crate::KeySpec::new`] + /// and [`crate::DialCache::invalidate`]. pub fn new( key_type: impl Into, - id: impl Into, + id: impl IntoKeyId, use_case: impl Into, ) -> Self { Identity { namespace: String::new(), key_type: key_type.into(), - id: id.into(), + id: id.into_key_id(), use_case: use_case.into(), tracked: false, args: Vec::new(), diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 7afbd9e4..ecbfe6a4 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -53,7 +53,7 @@ pub use engine::{ ScopeGuard, }; pub use error::{BoxError, ConfigError, Error, FallbackTimeout, RemoteReadTimeout, SharedError}; -pub use identity::{normalize_args, ArgValue, Identity, IdentityError, Keys}; +pub use identity::{normalize_args, ArgValue, Identity, IdentityError, IntoKeyId, Keys}; pub use local::{LocalEntry, LocalRead, LocalStore, LruLocalStore, StoredValue}; pub use metrics::MetricKind; pub use observe::{Event, Labels, LogEvent, LogLevel, Logger, Observer, ShadowMismatchDetails}; @@ -74,4 +74,4 @@ pub use runtime::Runtime; #[cfg(feature = "tokio")] pub use runtime::TokioRuntime; pub use scope::Scope; -pub use use_case::{IntoKeyId, KeySpec, UseCase, UseCaseBuilder}; +pub use use_case::{KeySpec, UseCase, UseCaseBuilder}; diff --git a/rust/src/testing.rs b/rust/src/testing.rs index 5da86f0f..ec0d742e 100644 --- a/rust/src/testing.rs +++ b/rust/src/testing.rs @@ -247,6 +247,7 @@ impl Runtime for StepRuntime { pub struct TestExecutor { pool: LocalPool, local: LocalSpawner, + polls: Arc, /// The virtual clock to share with every instance under test. pub clock: Arc, /// The runtime handle to pass to every instance under test. @@ -272,11 +273,27 @@ impl TestExecutor { TestExecutor { pool, local, + polls: Arc::new(AtomicU64::new(0)), clock, runtime, } } + /// Actual task polls performed by this executor. A zero-time verification + /// drain must leave this count unchanged when every task is already blocked. + pub fn poll_count(&self) -> u64 { + self.polls.load(Ordering::Relaxed) + } + + fn tracked(&self, future: F) -> impl Future + use { + let polls = self.polls.clone(); + let mut future = Box::pin(future); + futures::future::poll_fn(move |cx| { + polls.fetch_add(1, Ordering::Relaxed); + future.as_mut().poll(cx) + }) + } + /// Run a future to completion on the pool, draining detached work as it /// is spawned. Panics if the future stays blocked once nothing is runnable: /// it would be waiting on a gate only the test can release. @@ -285,10 +302,10 @@ impl TestExecutor { std::rc::Rc::new(std::cell::RefCell::new(None)); let sink = slot.clone(); self.local - .spawn_local(async move { + .spawn_local(self.tracked(async move { let result = future.await; *sink.borrow_mut() = Some(result); - }) + })) .expect("pool accepts tasks"); self.drain(); let result = slot.borrow_mut().take(); @@ -297,14 +314,18 @@ impl TestExecutor { /// Spawn driver-owned work onto the pool. pub fn spawn(&self, task: impl Future + 'static) { - self.local.spawn_local(task).expect("pool accepts tasks"); + self.local + .spawn_local(self.tracked(task)) + .expect("pool accepts tasks"); } fn move_immediate(&mut self) -> bool { let tasks = std::mem::take(&mut self.runtime.queues.lock().immediate); let moved = !tasks.is_empty(); for task in tasks { - self.local.spawn_local(task).expect("pool accepts tasks"); + self.local + .spawn_local(self.tracked(task)) + .expect("pool accepts tasks"); } moved } @@ -313,7 +334,9 @@ impl TestExecutor { let tasks = std::mem::take(&mut self.runtime.queues.lock().deferred); let moved = !tasks.is_empty(); for task in tasks { - self.local.spawn_local(task).expect("pool accepts tasks"); + self.local + .spawn_local(self.tracked(task)) + .expect("pool accepts tasks"); } moved } diff --git a/rust/src/use_case.rs b/rust/src/use_case.rs index dd42237b..30c82c11 100644 --- a/rust/src/use_case.rs +++ b/rust/src/use_case.rs @@ -11,7 +11,7 @@ use serde::Serialize; use crate::codec::{Codec, JsonCodec}; use crate::engine::DialCache; use crate::error::{BoxError, ConfigError, Error}; -use crate::identity::{js_number_to_string, normalize_args, ArgValue, Identity}; +use crate::identity::{normalize_args, ArgValue, Identity, IntoKeyId}; use crate::operation::{ downcast_value, erase_load, Comparator, ErasedOperation, Operation, Preview, RecoveryPredicate, SourceBudget, @@ -19,54 +19,6 @@ use crate::operation::{ use crate::policy::Policy; use crate::scope::Scope; -/// Converts an entity identifier to the text used in a cache key. -/// -/// Strings are preserved, integers use exact decimal notation, and floats use -/// JavaScript `String(number)` spelling (`f32` is promoted to `f64`). Shared -/// references to these types are also accepted. For other displayable IDs, -/// pass `id.to_string()` or implement this trait with the desired spelling. -pub trait IntoKeyId { - /// Consume the identifier and return its canonical text. - fn into_key_id(self) -> String; -} - -impl IntoKeyId for String { - fn into_key_id(self) -> String { - self - } -} - -impl IntoKeyId for &str { - fn into_key_id(self) -> String { - self.to_owned() - } -} - -impl IntoKeyId for &T { - fn into_key_id(self) -> String { - self.clone().into_key_id() - } -} - -macro_rules! integer_key_id { - ($($t:ty),*) => { $(impl IntoKeyId for $t { - fn into_key_id(self) -> String { self.to_string() } - })* }; -} -integer_key_id!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); - -impl IntoKeyId for f64 { - fn into_key_id(self) -> String { - js_number_to_string(self) - } -} - -impl IntoKeyId for f32 { - fn into_key_id(self) -> String { - js_number_to_string(f64::from(self)) - } -} - /// The key of one call: the entity id and any secondary dimensions. #[derive(Debug, Clone, PartialEq, Default)] pub struct KeySpec { diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index 15ab23c2..86764925 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -32,22 +32,6 @@ use formal::scenarios; use formal::transport::{Coordinator, Prepared}; use serde_json::{Map, Value}; -const FEATURE_PROFILES: [&str; 13] = [ - "admission", - "independent", - "layers", - "local-failure", - "policy", - "recovery", - "recovery-read", - "runtime-boundaries", - "scope", - "shadow", - "shadow-layers", - "source-budgets", - "runtime-boundaries-placeholder", -]; - struct Run { selection: Selection, report: Report, @@ -92,6 +76,13 @@ fn run() -> Result { profile_actions.insert(name.clone(), actions); } } + if let Some(selected) = &selection.feature_profile { + if !profile_actions.contains_key(selected) + || matches!(selected.as_str(), "core" | "effects") + { + return Err(format!("unknown selected feature profile: {selected}")); + } + } let mut run = Run { selection, report, @@ -169,8 +160,14 @@ impl Run { let mut apply = |input: &Value| cell.borrow_mut().apply(input); let mut observation = || cell.borrow().observation(); let mut wall = || cell.borrow().wall_ms(); - self.coordinator - .execute(&prepared, &mut apply, &mut observation, &mut wall, &mut []) + self.coordinator.execute( + &prepared, + &mut apply, + &mut observation, + &mut wall, + None, + &mut [], + ) } fn core(&mut self) -> Result<(), String> { @@ -198,7 +195,8 @@ impl Run { let cell = std::cell::RefCell::new(driver_ref); let mut apply = |input: &Value| cell.borrow_mut().apply(input); let mut observation = || cell.borrow().observation(); - let mut wall = || cell.borrow().wall_ms(); + let mut wall = || cell.borrow().observation_wall_ms(); + let mut receipt = || cell.borrow().receipt(); let mut monitor = || cell.borrow().assert_effects_history(); let mut monitors: Vec<&mut dyn FnMut() -> Result<(), String>> = Vec::new(); if effects_monitor { @@ -209,6 +207,7 @@ impl Run { &mut apply, &mut observation, &mut wall, + Some(&mut receipt), &mut monitors, ) }; @@ -285,6 +284,7 @@ impl Run { &mut apply, &mut observation, &mut wall, + None, &mut [], ) })(); @@ -400,11 +400,6 @@ impl Run { } } -#[allow(dead_code)] -fn _profiles() -> &'static [&'static str] { - &FEATURE_PROFILES -} - #[allow(dead_code)] fn _trace(source: &TraceSource) -> bool { source.is_directory() diff --git a/rust/tests/formal/driver.rs b/rust/tests/formal/driver.rs index 7fc4b5c7..9c3a59e7 100644 --- a/rust/tests/formal/driver.rs +++ b/rust/tests/formal/driver.rs @@ -135,7 +135,6 @@ pub struct Shared { fallback_failed: bool, pub discard_writes: bool, pub discard_invalidations: bool, - unsettled: Option>, } impl Shared { @@ -277,8 +276,11 @@ pub struct Driver { fixture: Value, instances: HashMap, scopes: HashMap, - /// Harness control only: report the observation held before the - /// settlement drain. Conformance replays never set it. + reported: Value, + settlement_receipt: Value, + reported_wall_ms: i64, + /// Harness control only: skip the first settlement drain and let the + /// verification drain attest to the work it finds still runnable. pub skip_settle: bool, } @@ -329,7 +331,6 @@ impl Driver { fallback_failed: false, discard_writes: false, discard_invalidations: false, - unsettled: None, }; let mut driver = Driver { exec, @@ -338,9 +339,13 @@ impl Driver { fixture, instances: HashMap::new(), scopes: HashMap::new(), + reported: Value::Null, + settlement_receipt: Value::Null, + reported_wall_ms: WALL_EPOCH_MS, skip_settle: false, }; driver.instance("default"); + driver.settle(); driver } @@ -627,6 +632,23 @@ impl Driver { fields.insert("ttlMs".to_string(), Value::from(ttl)); self.shared.lock().record("marker", fields); } + "inspectCoalescing" => { + let instance = input + .get("instance") + .and_then(Value::as_str) + .unwrap_or("default"); + let state = self.instance(instance).coalescing_state().process; + let fields = json!({ + "instance": instance, + "activeLeaders": state.active_leaders, + "activeFollowers": state.active_followers, + "oldestLeaderAgeMs": state.oldest_leader_age_ms, + }); + self.shared.lock().record( + "coalescingState", + fields.as_object().expect("fields").clone(), + ); + } "adapterReply" => { let mut s = self.shared.lock(); if s.reply.is_some() { @@ -680,15 +702,58 @@ impl Driver { } other => return Err(format!("unknown behavior input {other}: {input}")), } - if self.skip_settle { - let mut s = self.shared.lock(); - let snapshot = s.observed.clone(); - s.unsettled = Some(snapshot); + self.settle(); + self.assert_publication_causality() + } + + /// Attest to a single instant, then verify it with a second zero-time drain. + /// Counts come from actual executor polls and external gates, never predictions. + fn settle(&mut self) { + if !self.skip_settle { + self.exec.drain(); + } + let shared = self.shared.lock(); + self.reported = Value::Object(shared.observed.clone()); + let mut held = Map::new(); + held.insert( + "loaders".to_string(), + json!(shared + .loaders + .iter() + .filter(|gate| !gate.is_settled()) + .count()), + ); + for (kind, field) in [ + ("read", "reads"), + ("write", "writes"), + ("dump", "dumps"), + ("load", "loads"), + ("policy", "policies"), + ] { + held.insert( + field.to_string(), + json!(shared.effects.get(kind).map_or(0, HashMap::len)), + ); } - // Drain ready work while unresolved external gates remain held: the - // causally-ready-v1 settlement step. + held.insert( + "scopes".to_string(), + json!(self + .scopes + .values() + .filter(|scope| !scope.gate.is_settled()) + .count()), + ); + drop(shared); + let elapsed = self.elapsed_ms(); + self.reported_wall_ms = self.wall_ms(); + let polls = self.exec.poll_count(); + let timers = self.clock.pending_timers(); self.exec.drain(); - self.assert_publication_causality() + let changed = self.reported != Value::Object(self.shared.lock().observed.clone()); + let runnable = self.exec.poll_count() - polls + + u64::from(changed) + + timers.abs_diff(self.clock.pending_timers()) as u64; + self.settlement_receipt = json!({"elapsedMs": elapsed, "runnable": runnable, "held": held}); } fn begin(&mut self, input: &Value) -> Result<(), String> { @@ -933,15 +998,17 @@ impl Driver { Ok(()) } - /// The complete current observation. + /// The reported observation at the instant its receipt describes. pub fn observation(&self) -> Value { - let s = self.shared.lock(); - if self.skip_settle { - if let Some(unsettled) = &s.unsettled { - return Value::Object(unsettled.clone()); - } - } - Value::Object(s.observed.clone()) + self.reported.clone() + } + + pub fn receipt(&self) -> Value { + self.settlement_receipt.clone() + } + + pub fn observation_wall_ms(&self) -> i64 { + self.reported_wall_ms } /// Release every driver-owned gate so no work leaks into the next history. diff --git a/rust/tests/formal/inventory.rs b/rust/tests/formal/inventory.rs index edd392ae..84f88ed7 100644 --- a/rust/tests/formal/inventory.rs +++ b/rust/tests/formal/inventory.rs @@ -235,11 +235,14 @@ pub fn glob_itf(directory: &Path) -> Result, String> { Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), Err(error) => return Err(format!("{}: {error}", directory.display())), }; - let mut names: Vec = entries - .filter_map(Result::ok) - .map(|entry| entry.file_name().to_string_lossy().into_owned()) - .filter(|name| name.ends_with(".itf.json")) - .collect(); + let mut names = Vec::new(); + for entry in entries { + let entry = entry.map_err(|error| format!("{}: {error}", directory.display()))?; + let name = entry.file_name().to_string_lossy().into_owned(); + if name.ends_with(".itf.json") { + names.push(name); + } + } names.sort(); Ok(names.into_iter().map(|name| directory.join(name)).collect()) } @@ -256,56 +259,19 @@ fn parent_of(directory: &Path) -> PathBuf { parent } -/// Scheduled Quint regressions of `profile`, expected at -/// `/../regressions//.itf.json`. +/// Exported Quint regressions of `profile`, read from +/// `/../regressions//`. The shared inventory derives which +/// runs export from the Quint source and the full-report gate requires exactly +/// those histories; the native driver only discovers their generated files. pub fn regression_paths(profile: &str, directory: &Path) -> Result, String> { let regressions = parent_of(directory).join("regressions").join(profile); - let mut paths = Vec::new(); - for name in replay_regressions(profile)? { - let path = regressions.join(format!("{name}.itf.json")); - if let Err(error) = std::fs::metadata(&path) { - return Err(format!( - "missing Quint regression {name}: {}: {error}", - path.display() - )); - } - paths.push(path); + let paths = glob_itf(®ressions)?; + if paths.is_empty() { + return Err(format!("missing Quint regressions for {profile}")); } Ok(paths) } -/// `replayRegressions` of the `formal/execution.json` model bound to `profile`. -pub fn replay_regressions(profile: &str) -> Result, String> { - let manifest = read_json("formal/execution.json")?; - let mut names = Vec::new(); - for model in manifest - .get("models") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - if model.get("profile").and_then(Value::as_str) != Some(profile) { - continue; - } - for name in model - .get("replayRegressions") - .and_then(Value::as_array) - .into_iter() - .flatten() - { - match name.as_str() { - Some(name) => names.push(name.to_string()), - None => { - return Err( - "malformed replayRegressions entry in formal/execution.json".to_string() - ) - } - } - } - } - Ok(names) -} - fn read_json(relative: &str) -> Result { let path = repo_path(relative); let text = @@ -385,14 +351,14 @@ pub fn percent_encode_component(text: &str) -> String { /// Behavior profile versions this port implements, from /// `go/behavior_registry_test.go`. -pub const BEHAVIOR_PROFILE_VERSIONS: [(&str, i64); 14] = [ +pub const BEHAVIOR_PROFILE_VERSIONS: [(&str, i64); 16] = [ ("recovery-read", 1), ("local-failure", 1), ("runtime-boundaries", 1), ("shadow-layers", 1), ("local-clock", 1), ("source-budgets", 1), - ("effects", 2), + ("effects", 3), ("scope", 2), ("policy", 3), ("layers", 2), @@ -400,6 +366,8 @@ pub const BEHAVIOR_PROFILE_VERSIONS: [(&str, i64); 14] = [ ("independent", 2), ("shadow", 3), ("admission", 1), + ("dark-layers", 2), + ("shadow-read-deadlines", 1), ]; /// The version this port implements of a behavior profile, if it knows it. diff --git a/rust/tests/formal/transport.rs b/rust/tests/formal/transport.rs index d92c74ee..234b044e 100644 --- a/rust/tests/formal/transport.rs +++ b/rust/tests/formal/transport.rs @@ -56,6 +56,8 @@ pub struct Prepared { pub steps: i64, /// `$defs` definition every observation of this session must satisfy. pub observation: String, + /// Receipt definition for a behavior session; absent for core/local-clock. + pub receipt: Option, /// Initialization fixture for the native driver (an object). pub fixture: Value, /// Commands to apply before the first observation. @@ -269,7 +271,8 @@ impl Coordinator { let result = self.call(request)?; let result = result.as_object().ok_or("malformed replay preparation")?; let session = result.get("session").and_then(Value::as_str).unwrap_or(""); - if sorted_keys(result) != "actions,fixture,observation,session,settlement,setup,steps" + if sorted_keys(result) + != "actions,fixture,observation,receipt,session,settlement,setup,steps" || result.get("settlement").and_then(Value::as_str) != Some(SETTLEMENT) || session.is_empty() || !safe_index(result.get("steps"), 2) @@ -287,6 +290,17 @@ impl Coordinator { "malformed replay observation definition {definition:?}" )); } + let receipt = match result.get("receipt") { + Some(Value::Null) if definition != "behaviorObservation" => None, + Some(Value::String(name)) + if name == "settlementReceipt" + && definition == "behaviorObservation" + && self.schema.definition(name).is_some() => + { + Some(name.clone()) + } + _ => return Err("malformed replay receipt definition".to_string()), + }; let fixture = match result.get("fixture") { Some(fixture @ Value::Object(_)) => fixture.clone(), _ => return Err("malformed replay fixture".to_string()), @@ -312,6 +326,7 @@ impl Coordinator { session: session.to_string(), steps, observation: definition.to_string(), + receipt, fixture, setup, actions: names, @@ -328,9 +343,10 @@ impl Coordinator { apply: &mut dyn FnMut(&Value) -> Result<(), String>, observation: &mut dyn FnMut() -> Value, wall_ms: &mut dyn FnMut() -> i64, + receipt: Option<&mut dyn FnMut() -> Value>, monitors: &mut [&mut dyn FnMut() -> Result<(), String>], ) -> Result<(), String> { - let outcome = self.run(prepared, apply, observation, wall_ms, monitors); + let outcome = self.run(prepared, apply, observation, wall_ms, receipt, monitors); if outcome.is_err() { let mut request = Map::new(); request.insert("op".to_string(), Value::from("discard")); @@ -349,8 +365,17 @@ impl Coordinator { apply: &mut dyn FnMut(&Value) -> Result<(), String>, observation: &mut dyn FnMut() -> Value, wall_ms: &mut dyn FnMut() -> i64, + mut receipt: Option<&mut dyn FnMut() -> Value>, monitors: &mut [&mut dyn FnMut() -> Result<(), String>], ) -> Result<(), String> { + if prepared.receipt.is_some() != receipt.is_some() { + return Err(if prepared.receipt.is_some() { + "driver omitted its settlement receipt callback" + } else { + "driver supplied an unexpected settlement receipt callback" + } + .to_string()); + } for input in &prepared.setup { apply(input)?; } @@ -376,6 +401,17 @@ impl Coordinator { request.insert("settlement".to_string(), Value::from(SETTLEMENT)); request.insert("observed".to_string(), observed); request.insert("environment".to_string(), Value::Object(environment)); + if let Some(callback) = receipt.as_mut() { + let value = callback(); + let definition = prepared + .receipt + .as_deref() + .expect("validated receipt callback"); + if !self.schema.matches_definition(&value, definition) { + return Err(format!("driver produced a malformed {definition} receipt")); + } + request.insert("receipt".to_string(), value); + } let result = self.call(request)?; let result = result.as_object().ok_or("malformed next replay command")?; let next = index + 1; diff --git a/rust/tests/formal/witness.rs b/rust/tests/formal/witness.rs index 71d24258..24c1e4be 100644 --- a/rust/tests/formal/witness.rs +++ b/rust/tests/formal/witness.rs @@ -10,7 +10,7 @@ use super::inventory::trace_kind; use super::json::strict_parse; use serde::{Deserialize, Deserializer}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; /// Hex SHA-256 of a file's bytes. @@ -200,7 +200,20 @@ pub fn check_witness_evidence( let execution = read_strict(&root.join("formal/execution.json"))?; let definitions = read_strict(&root.join("formal/profiles.json"))?; let shared = shared_replay_sources(root)?; - let mut additional: Vec = strings_at(&execution, "libraries")?; + let claimed: HashSet = execution + .get("models") + .and_then(serde_json::Value::as_array) + .into_iter() + .flatten() + .map(|model| { + model + .get("path") + .and_then(serde_json::Value::as_str) + .map(str::to_owned) + .ok_or_else(|| "execution model must name its path".to_string()) + }) + .collect::>()?; + let mut additional = quint_libraries(root, &claimed)?; additional.extend(shared); for definition in definitions .get("profiles") @@ -248,6 +261,38 @@ pub fn check_witness_evidence( Ok(()) } +/// Every Quint source in `formal/` and `formal/kernel/` not claimed by a +/// scheduled model, sorted as `formal/execution.mjs` derives library inputs. +pub fn quint_libraries(root: &Path, claimed: &HashSet) -> Result, String> { + let mut libraries = Vec::new(); + for folder in ["formal", "formal/kernel"] { + let directory = root.join(folder); + let entries = match std::fs::read_dir(&directory) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, + Err(error) => return Err(format!("{}: {error}", directory.display())), + }; + for entry in entries { + let entry = entry.map_err(|error| format!("{}: {error}", directory.display()))?; + let name = entry.file_name().to_string_lossy().into_owned(); + if entry + .file_type() + .map_err(|error| format!("{}: {error}", entry.path().display()))? + .is_dir() + || !name.ends_with(".qnt") + { + continue; + } + let path = format!("{folder}/{name}"); + if !claimed.contains(&path) { + libraries.push(path); + } + } + } + libraries.sort(); + Ok(libraries) +} + fn strings_at(value: &serde_json::Value, key: &str) -> Result, String> { match value.get(key) { None | Some(serde_json::Value::Null) => Ok(Vec::new()), diff --git a/rust/tests/harness_infra.rs b/rust/tests/harness_infra.rs index 4a837d30..d2a8d361 100644 --- a/rust/tests/harness_infra.rs +++ b/rust/tests/harness_infra.rs @@ -348,6 +348,7 @@ fn prepared_control() -> Prepared { session: "1".to_string(), steps: 2, observation: "coreObservation".to_string(), + receipt: None, fixture: Value::Object(Map::new()), setup: Vec::new(), actions: vec!["init".to_string(), "outsideCall".to_string()], @@ -396,7 +397,7 @@ fn coordinator_answers_repeated_profile_requests() { .cloned() .expect("profiles object"); } - assert_eq!(profiles.len(), 15, "profiles: {}", sorted_keys(&profiles)); + assert_eq!(profiles.len(), 17, "profiles: {}", sorted_keys(&profiles)); for name in [ "core", "effects", @@ -543,6 +544,7 @@ fn coordinator_rejects_premature_completion_and_empty_commands() { }, &mut healthy_core_observation, &mut || 0, + None, &mut [], ); assert!( @@ -670,6 +672,7 @@ fn transport_validates_observations_locally() { }, &mut || with(&healthy_core_observation(), "redisReads", json!("many")), &mut || 0, + None, &mut [], ) .expect_err("malformed observation accepted"); @@ -683,7 +686,7 @@ fn transport_validates_observations_locally() { .expect("an observe request reached the coordinator"); // prepare rejects an unknown observation definition. - let program = "require('readline').createInterface({input:process.stdin}).on('line', line => { const request=JSON.parse(line); process.stdout.write(JSON.stringify({version:1,id:request.id,ok:true,result:{session:'1',settlement:'causally-ready-v1',observation:'observedEvent',fixture:{},setup:[],actions:['init','outsideCall'],steps:2}})+'\\n'); });"; + let program = "require('readline').createInterface({input:process.stdin}).on('line', line => { const request=JSON.parse(line); process.stdout.write(JSON.stringify({version:1,id:request.id,ok:true,result:{session:'1',settlement:'causally-ready-v1',observation:'observedEvent',receipt:null,fixture:{},setup:[],actions:['init','outsideCall'],steps:2}})+'\\n'); });"; let mut coordinator = Coordinator::start(node(program), Duration::from_secs(1), false).expect("start"); let error = coordinator @@ -701,6 +704,90 @@ fn carries_comparison_markers(text: &str) -> bool { .is_some_and(|at| text[at..].contains("actual:")) } +#[test] +fn transport_validates_settlement_receipts_before_sending() { + let receipt = json!({"elapsedMs": 0, "runnable": 0, "held": { + "loaders": 0, "reads": 0, "writes": 0, "dumps": 0, + "loads": 0, "policies": 0, "scopes": 0 + }}); + let schema = Schema::load().expect("schema"); + assert!(schema.matches_definition(&receipt, "settlementReceipt")); + let mut bad_held = receipt.clone(); + bad_held["held"]["reads"] = json!(-1); + let malformed = [ + without(&receipt, "held"), + with(&receipt, "runnable", json!(0.5)), + with(&receipt, "elapsedMs", json!("0")), + with(&receipt, "extra", json!(0)), + bad_held, + ]; + // The process rejects any observe request; a malformed receipt must be + // attributed to the driver before it spends a coordinator round trip. + let program = "require('readline').createInterface({input:process.stdin}).on('line', line => { const r=JSON.parse(line); if (r.op === 'observe') process.exit(3); process.stdout.write(JSON.stringify({version:1,id:r.id,ok:true,result:{discarded:true}})+'\\n'); });"; + let mut coordinator = + Coordinator::start(node(program), Duration::from_secs(1), false).expect("start"); + let mut prepared = prepared_control(); + prepared.observation = "behaviorObservation".to_string(); + prepared.receipt = Some("settlementReceipt".to_string()); + for malformed in malformed { + let error = coordinator + .execute( + &prepared, + &mut |_| Ok(()), + &mut || empty_behavior_observation(true), + &mut || 0, + Some(&mut || malformed.clone()), + &mut [], + ) + .expect_err("malformed receipt accepted"); + assert!( + error.contains("malformed settlementReceipt receipt"), + "{error}" + ); + assert!(!carries_comparison_markers(&error), "{error}"); + } + let missing = coordinator + .execute( + &prepared, + &mut |_| Ok(()), + &mut || empty_behavior_observation(true), + &mut || 0, + None, + &mut [], + ) + .expect_err("missing receipt callback accepted"); + assert!( + missing.contains("omitted its settlement receipt"), + "{missing}" + ); + coordinator + .finish() + .expect("an observe request reached the coordinator"); + + // prepare must reject invalid receipt definitions and prevent a core + // session from attaching a behavior driver's receipt. + for (observation, value) in [ + ("behaviorObservation", Value::Null), + ("behaviorObservation", json!("coreObservation")), + ("coreObservation", json!("settlementReceipt")), + ] { + let result = json!({"session":"1", "settlement":SETTLEMENT, + "observation":observation, "receipt":value, "fixture":{}, + "setup":[], "actions":["init","outsideCall"], "steps":2}); + let program = format!("require('readline').createInterface({{input:process.stdin}}).on('line', line => {{ const r=JSON.parse(line); process.stdout.write(JSON.stringify({{version:1,id:r.id,ok:true,result:{result}}})+'\\n'); }});"); + let mut coordinator = + Coordinator::start(node(&program), Duration::from_secs(1), false).expect("start"); + let error = coordinator + .prepare("core", Path::new("control.itf.json"), None) + .expect_err("invalid receipt definition accepted"); + assert!( + error.contains("malformed replay receipt definition"), + "{error}" + ); + coordinator.finish().expect("clean exit"); + } +} + #[test] fn coordinator_prepares_and_replays_the_smoke_history_shape() { // The real coordinator's prepare result must satisfy every structural check. @@ -893,8 +980,8 @@ fn missing_scheduled_regressions_fail_corpus_selection() { regression_paths("effects", &corpus.0).expect_err("missing scheduled histories accepted"); assert!(error.contains("missing Quint regression"), "{error}"); assert!(regression_paths("not-a-profile", &corpus.0) - .expect("no regressions") - .is_empty()); + .expect_err("unknown profile without exports accepted") + .contains("missing Quint regression")); } #[test] @@ -1058,7 +1145,10 @@ fn witness_evidence_binds_shared_replay_sources() { "formal/coverage-witnesses.json", r#"{"effects":["observed"]}"#, ); - root.write("formal/execution.json", r#"{"libraries":[]}"#); + root.write( + "formal/execution.json", + r#"{"models":[{"path":"formal/dialcache-effects-conformance.qnt"}]}"#, + ); root.write("trace.itf.json", "controlled trace"); let trace = root.0.join("trace.itf.json"); let cite = |name: &str, kind: &str, checkpoints: Vec| Trace { @@ -1207,6 +1297,36 @@ fn witness_evidence_binds_shared_replay_sources() { write(&evidence); check().expect("restored evidence rejected"); + // Library membership is discovered from actual Quint sources, so adding + // a kernel library must invalidate the old evidence even without a + // manifest edit; updating its digest then makes subsequent drift visible. + root.write("formal/kernel/new-library.qnt", "new reviewed library"); + let error = check().expect_err("new kernel library omitted from evidence"); + assert!( + error.contains("incomplete witness definition fingerprints"), + "{error}" + ); + let mut with_library = evidence.clone(); + with_library.inputs.insert( + 5, + Digest { + path: "formal/kernel/new-library.qnt".to_string(), + name: String::new(), + sha256: file_sha256(&root.0.join("formal/kernel/new-library.qnt")).expect("hash"), + }, + ); + write(&with_library); + check().expect("new library fingerprint rejected"); + root.write("formal/kernel/new-library.qnt", "changed library"); + let error = check().expect_err("stale kernel library digest accepted"); + assert!( + error.contains("stale witness definition formal/kernel/new-library.qnt"), + "{error}" + ); + std::fs::remove_file(root.0.join("formal/kernel/new-library.qnt")).expect("remove library"); + write(&evidence); + check().expect("restored library inventory rejected"); + root.write("formal/replay/mapping.mjs", "changed input mapping"); let error = check().expect_err("changed shared mapping accepted"); assert!( @@ -1241,6 +1361,7 @@ fn observe_requests_reach_the_coordinator_comparison() { }, &mut healthy_core_observation, &mut || 1_788_868_800_000, + None, &mut [], ) .expect_err("a zero observation matched the smoke history"); diff --git a/rust/tests/key_spec.rs b/rust/tests/key_spec.rs index 6f8e678d..615433e3 100644 --- a/rust/tests/key_spec.rs +++ b/rust/tests/key_spec.rs @@ -1,15 +1,16 @@ -//! Native entity-ID conversion at the registered-use-case API boundary. +//! Entity-ID conversion shared by registered use cases, inline calls and invalidation. // Borrowed input support is part of the public contract exercised below. #![allow(clippy::needless_borrows_for_generic_args)] +use std::borrow::Cow; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; use dialcache::{ - BoxError, DialCache, IntoKeyId, InvalidateRequest, KeySpec, MissReason, Policy, ReadContext, - ReadRequest, ReadResult, Remote, WriteRequest, + BoxError, DialCache, Identity, IntoKeyId, InvalidateRequest, KeySpec, MissReason, Operation, + Policy, ReadContext, ReadRequest, ReadResult, Remote, WriteRequest, }; use futures::future::BoxFuture; use parking_lot::Mutex; @@ -28,6 +29,8 @@ fn floating_ids_use_javascript_spelling_for_owned_and_borrowed_inputs() { assert_eq!(KeySpec::new(value).id, expected); assert_eq!(KeySpec::new(&value).id, expected); assert_eq!(KeySpec::from(value).id, expected); + assert_eq!(Identity::new("thing", value, "byId").id, expected); + assert_eq!(Identity::new("thing", &value, "byId").id, expected); } } @@ -37,6 +40,8 @@ fn single_precision_ids_are_promoted_to_javascript_numbers() { assert_eq!(KeySpec::new(value).id, expected); assert_eq!(KeySpec::new(&value).id, expected); assert_eq!(KeySpec::from(value).id, expected); + assert_eq!(Identity::new("thing", value, "byId").id, expected); + assert_eq!(Identity::new("thing", &value, "byId").id, expected); } } @@ -60,6 +65,35 @@ fn string_and_integer_ids_keep_their_text_and_support_references() { "127.0.0.1" ); assert_eq!(42_u64.into_key_id(), "42"); + assert_eq!(Identity::new("thing", text.as_str(), "byId").id, text); + assert_eq!(Identity::new("thing", &borrowed_text, "byId").id, text); + assert_eq!(Identity::new("thing", &text, "byId").id, text); + assert_eq!(Identity::new("thing", text.clone(), "byId").id, text); + assert_eq!( + Identity::new("thing", text.into_boxed_str(), "byId").id, + "001e+21" + ); + assert_eq!(Identity::new("thing", Cow::Borrowed("-0"), "byId").id, "-0"); + assert_eq!( + Identity::new("thing", Cow::<'_, str>::Owned("1e+21".to_owned()), "byId").id, + "1e+21" + ); + assert_eq!(Identity::new("thing", 'é', "byId").id, "é"); +} + +#[test] +fn direct_identities_preserve_every_primitive_integer_domain() { + macro_rules! check { + ($($kind:ty),*) => { $( + for value in [0 as $kind, <$kind>::MIN, <$kind>::MAX] { + let expected = value.to_string(); + assert_eq!(Identity::new("thing", value, "byId").id, expected); + assert_eq!(Identity::new("thing", &value, "byId").id, expected); + assert_eq!(KeySpec::new(value).id, expected); + } + )* }; + } + check!(i8, i16, i32, i64, i128, isize, u8, u16, u32, u64, u128, usize); } #[test] @@ -82,13 +116,20 @@ fn registered_float_ids_share_the_javascript_zero_identity() { }) .register() .unwrap(); - let (negative, positive) = executor.block_on(async move { + let (negative, positive, inline) = executor.block_on(async move { let request = cache.enable_guard(); let negative = lookup.get(request.scope(), -0.0).await.unwrap(); let positive = lookup.get(request.scope(), 0.0).await.unwrap(); - (negative, positive) + let operation = Operation::::new(Identity::new("thing", -0.0, "FloatId")) + .policy(Policy::default().local_ttl_sec(60)); + let inline = cache + .get_or_load(request.scope(), operation, |_| async { Ok(999) }) + .await + .unwrap(); + (negative, positive, inline) }); assert_eq!((*negative, *positive), (1, 1)); + assert!(Arc::ptr_eq(&negative, &inline)); assert_eq!(sources.load(Ordering::SeqCst), 1); } @@ -139,16 +180,32 @@ where .source(|_, _| async { Ok(7) }) .register() .unwrap(); + let direct = Identity::new("thing", &id, "TrackedId") + .namespace("numeric") + .tracked(true); + let keys = direct.keys().unwrap(); + assert_eq!(direct.id, KeySpec::new(&id).id); executor.block_on(async move { let request = cache.enable_guard(); assert_eq!(*lookup.get(request.scope(), id.clone()).await.unwrap(), 7); + let operation = Operation::::new(direct).policy(Policy::default().remote_ttl_sec(60)); + assert_eq!( + *cache + .get_or_load(request.scope(), operation, |_| async { Ok(8) }) + .await + .unwrap(), + 8 + ); cache.invalidate("thing", &id, 17).await.unwrap(); cache.invalidate("thing", id, 17).await.unwrap(); }); let expected = format!("{{numeric:thing:{escaped_id}}}#watermark"); let reads = remote.reads.lock(); - assert_eq!(reads.len(), 1); + assert_eq!(reads.len(), 2); + assert_eq!(reads[0], reads[1]); + assert_eq!(reads[0].value_key, keys.value); + assert_eq!(reads[0].watermark_key, keys.watermark); assert_eq!(reads[0].watermark_key.as_deref(), Some(expected.as_str())); let invalidations = remote.invalidations.lock(); assert_eq!(invalidations.len(), 2); diff --git a/rust/tests/settlement_control.rs b/rust/tests/settlement_control.rs index 4f5e62c2..297b033b 100644 --- a/rust/tests/settlement_control.rs +++ b/rust/tests/settlement_control.rs @@ -1,16 +1,16 @@ -//! The no-settle control: a driver that reports the observation held before -//! the settlement drain must fail the observation assertions of every -//! behavior-driver-backed smoke history. It proves the replays depend on the -//! `causally-ready-v1` settlement. +//! The no-settle control: skipping the driver drain must fail the settlement +//! contract of every behavior smoke history before observation comparison. mod formal; use formal::driver::{install_panic_hook, Driver}; use formal::inventory::repo_path; use formal::transport::Coordinator; -use serde_json::Value; +use serde_json::{json, Value}; -const BEHAVIOR_SMOKE: [&str; 13] = [ +const BEHAVIOR_SMOKE: [&str; 15] = [ + "dark-layers", + "shadow-read-deadlines", "effects", "admission", "independent", @@ -42,8 +42,16 @@ fn replay(coordinator: &mut Coordinator, profile: &str, skip_settle: bool) -> Re let cell = std::cell::RefCell::new(&mut driver); let mut apply = |input: &Value| cell.borrow_mut().apply(input); let mut observation = || cell.borrow().observation(); - let mut wall = || cell.borrow().wall_ms(); - coordinator.execute(&prepared, &mut apply, &mut observation, &mut wall, &mut []) + let mut wall = || cell.borrow().observation_wall_ms(); + let mut receipt = || cell.borrow().receipt(); + coordinator.execute( + &prepared, + &mut apply, + &mut observation, + &mut wall, + Some(&mut receipt), + &mut [], + ) }; driver.close(); result @@ -63,9 +71,37 @@ fn unsettled_observations_fail_every_behavior_smoke_history() { ); let message = unsettled.unwrap_err(); assert!( - message.contains("expected:") && message.contains("actual:"), - "{profile}: control failed without comparison evidence: {message}" + message.contains("Settlement violation:") + && !message.contains("expected:") + && !message.contains("actual:"), + "{profile}: control did not fail its own settlement contract: {message}" ); } coordinator.finish().expect("coordinator exit"); } + +#[test] +fn receipt_detects_runnable_work_without_observation_changes() { + let mut driver = Driver::new(json!({})); + let before = driver.observation(); + let clock = driver.observation_wall_ms(); + let finished = std::rc::Rc::new(std::cell::Cell::new(false)); + let flag = finished.clone(); + driver.exec.spawn(async move { flag.set(true) }); + driver.skip_settle = true; + driver + .apply(&json!({"op":"faults", "value":{}})) + .expect("command"); + assert!(finished.get(), "verification drain must run ready work"); + assert_eq!(driver.observation(), before, "work has no observed effects"); + assert_eq!( + driver.observation_wall_ms(), + clock, + "draining must consume no time" + ); + assert!( + driver.receipt()["runnable"].as_u64().unwrap() > 0, + "the receipt must detect actual polls even when observations and timers do not change" + ); + driver.close(); +} diff --git a/test/formal-mutation-shards.test.ts b/test/formal-mutation-shards.test.ts index 21d98010..d940affd 100644 --- a/test/formal-mutation-shards.test.ts +++ b/test/formal-mutation-shards.test.ts @@ -631,22 +631,39 @@ describe("Rust mutation language", () => { expect(fingerprintFiles(directory, ["rust/Cargo.toml"]).files).toBe(1); }); - it("merges Rust shards with the same four-cohort strictness as Go and renders the Rust report", () => { - const catalogSha256 = sha256(readRepo("formal/mutations.json")); - const inputs = { files: 3, sha256: "dd".repeat(32) }; - const single = { ...goSingleReport(catalogSha256, inputs), cargo: "cargo 1.98.1 (797e8a9bc 2026-08-05)" } as Report; - delete (single as Record).go; - const merged = mergeShardReports(languages.rust, goShardReports(single, 3), { catalog: goCatalog, catalogSha256, inputs }); - gateDetections(languages.rust, merged, goCatalog.mutations); + it("merges the separate Rust catalog without claiming shared model-boundary evidence", () => { + const catalogBytes = readRepo("formal/rust-mutations.json"); + const catalog = JSON.parse(catalogBytes.toString()) as Catalog; + for (const path of languages.rust.inputs) mkdirSync(join(directory, path), { recursive: true }); + writeFileSync(join(directory, languages.rust.catalog), catalogBytes); + const catalogSha256 = sha256(catalogBytes); + const inputs = fingerprintFiles(directory, languages.rust.inputs, { exclude: languages.rust.exclude }); + const mutations = catalog.mutations.map(entry => { + const { boundary: _boundary, ...result } = goMutation(entry); + return result; + }); + const single = { schemaVersion: 1, complete: true, startedAt: "2026-09-21T00:00:00.000Z", elapsedSeconds: 1, + cargo: "cargo 1.98.1", catalogSha256, inputs, baselines: goBaselines(), mutations, + scope: { catalog: "rust-native", modelBoundaryEvidence: false, unmappedSharedMutations: ["M14"] } } as Report; + const shards = goShardReports(single, 3); + for (const report of shards) { + const path = join(directory, languages.rust.output, "shards", `${report.shard!.index}-of-3`); + mkdirSync(path, { recursive: true }); + writeFileSync(join(path, "report.json"), JSON.stringify(report)); + } + const merged = mergeMutationReports("rust", { directory }); expect(merged.complete).toBe(true); - expect(merged.detection).toEqual(goDetection(single.mutations)); + expect(merged.detection).toEqual(goDetection(mutations)); expect(merged.requiredDetectionRegressions).toEqual([]); - expect(languages.rust.output).toBe(".formal-traces/rust-semantic"); - expect(languages.rust.catalog).toBe("formal/rust-mutations.json"); - const markdown = languages.rust.markdown(merged); + expect(merged.scope).toEqual(single.scope); + const markdown = readFileSync(join(directory, languages.rust.output, "report.md"), "utf8"); expect(markdown).toContain("# Rust semantic mutation measurement"); expect(markdown).toContain("| M01 | C45.maximum-age-exclusive | survived | detected | detected | detected |"); + expect(markdown).toContain("model-challenge boundary coverage is not measured"); expect(markdown).toContain("Merged from 3 shards"); + shards[0]!.mutations[0]!.cohorts.generated!.state = "survived"; + writeFileSync(join(directory, languages.rust.output, "shards/1-of-3/report.json"), JSON.stringify(shards[0])); + expect(() => mergeMutationReports("rust", { directory })).toThrow(/M01\/generated/); }); }); diff --git a/test/formal-rust-replay.test.ts b/test/formal-rust-replay.test.ts index ca9a87ce..d30447ed 100644 --- a/test/formal-rust-replay.test.ts +++ b/test/formal-rust-replay.test.ts @@ -10,7 +10,7 @@ type Step = { label: string; command?: string; args?: string[]; cwd?: string; en type Record_ = Record; const { checkRustReplay } = await import(new URL("../formal/check-rust-replay.mjs", import.meta.url).href) as { - checkRustReplay(report: string, inventory: Entry[]): Summary; + checkRustReplay(report: string, inventory?: Entry[]): Summary; }; const { parseRustReport, adaptReport, nativeBinding } = await import(new URL("../formal/conformance-adapters.mjs", import.meta.url).href) as { parseRustReport(text: string, inventory: Entry[]): { startedAt: number; finishedAt: number; results: Array<{ id: string; status: string }> }; @@ -58,6 +58,20 @@ describe("Rust replay report gate", () => { for (const profile of Object.keys(summary.generated)) expect(summary.quintRegressions, profile).toHaveProperty(profile); }); + it("requires new profiles and source-derived exported regressions through the default inventory", () => { + for (const profile of ["dark-layers", "shadow-read-deadlines"]) { + for (const category of ["sampled", "regression", "witness"]) { + expect(inventory.some(entry => entry.profile === profile && entry.category === category), `${profile}/${category}`).toBe(true); + } + const oldCases = inventory.filter(entry => entry.profile !== profile); + const oldReport = encode([start(), ...oldCases.map(entry => caseRecord(entry.id)), finish(oldCases.length)]); + expect(() => checkRustReplay(oldReport)).toThrow(`Missing passed Rust replay case: sampled/${profile}/0`); + } + const historiesOnly = inventory.filter(entry => entry.category !== "regression"); + const missingRegressions = encode([start(), ...historiesOnly.map(entry => caseRecord(entry.id)), finish(historiesOnly.length)]); + expect(() => checkRustReplay(missingRegressions)).toThrow(/Missing passed Rust replay case: regression\//); + }); + it("rejects a missing, duplicate or failed case", () => { const missing = inventory.at(-1)!.id; expect(() => checkRustReplay(withCases(records => { records.splice(records.length - 2, 1); (records.at(-1) as { cases: number }).cases--; }), inventory)) diff --git a/test/formal-rust-semantic-runner.test.ts b/test/formal-rust-semantic-runner.test.ts index dbc10c22..c3a3da1e 100644 --- a/test/formal-rust-semantic-runner.test.ts +++ b/test/formal-rust-semantic-runner.test.ts @@ -3,10 +3,12 @@ import { describe, expect, it } from "vitest"; const moduleUrl = new URL("../formal/measure-rust-semantics.mjs", import.meta.url).href; type Cohort = { state: string; passed: number; failed: number; failingTests: string[]; executedTests: string[]; assertionKinds: Record; assertionEvidence: Record }; -const { evaluateCargoTestOutput, evaluateRustReport, infrastructureTestFile } = await import(moduleUrl) as { +const { evaluateCargoTestOutput, evaluateRustReport, infrastructureTestFile, rustMutationScope, rustTargetDirectory } = await import(moduleUrl) as { evaluateCargoTestOutput(output: string, exitCode: number, expectedBinaries?: number): Cohort; evaluateRustReport(text: string, exitCode: number, stderr?: string): Cohort; infrastructureTestFile: RegExp; + rustMutationScope(catalog: unknown, typescript: unknown[]): { catalog: string; modelBoundaryEvidence: boolean; mappedMutations: string[]; unmappedSharedMutations: string[] }; + rustTargetDirectory(directory: string, selection: { shard: { index: number; count: number }; only?: string[] }): string; }; // libtest output as `cargo test --release --no-fail-fast --lib --test tokio_runtime` prints it. @@ -129,17 +131,23 @@ describe("Rust fault catalog", () => { const readRepo = (path: string) => readFileSync(new URL(`../${path}`, import.meta.url), "utf8"); type Entry = { id: string; case: string; description: string; typescriptMutation: string; edits: { path: string; before: string; after: string }[]; requiredDetections: string[]; typescriptRequiredDetections: string[] }; const catalog = JSON.parse(readRepo("formal/rust-mutations.json")) as { schemaVersion: number; mutations: Entry[] }; - const typescript = JSON.parse(readRepo("formal/semantic-mutations.json")) as { mutations: { id: string; case: string; description: string; requiredDetections: string[] }[] }; + const typescript = JSON.parse(readRepo("formal/mutations.json")) as { mutations: { id: string; case: string; description: string; typescript: { requiredDetections: string[] } }[] }; - it("names every TypeScript fault once with the same case and description", () => { + it("maps each scoped Rust fault once into the current shared catalog", () => { expect(catalog.schemaVersion).toBe(1); - expect(catalog.mutations.map(m => m.id)).toEqual(typescript.mutations.map(m => m.id)); + const scope = rustMutationScope(catalog, typescript.mutations); + expect(scope).toMatchObject({ catalog: "rust-native", modelBoundaryEvidence: false }); + expect(scope.mappedMutations).toEqual(catalog.mutations.map(m => m.typescriptMutation)); + expect([...scope.mappedMutations, ...scope.unmappedSharedMutations].sort()).toEqual(typescript.mutations.map(m => m.id).sort()); + expect(scope.unmappedSharedMutations).toContain("M14"); + expect(() => rustMutationScope({ ...catalog, mutations: [...catalog.mutations, catalog.mutations[0]] }, typescript.mutations)).toThrow(/duplicate/); + expect(() => rustMutationScope(catalog, typescript.mutations.slice(1))).toThrow(/counterpart/); for (const entry of catalog.mutations) { const counterpart = typescript.mutations.find(m => m.id === entry.typescriptMutation)!; expect(counterpart, entry.id).toBeDefined(); expect(entry.case, entry.id).toBe(counterpart.case); expect(entry.description, entry.id).toBe(counterpart.description); - expect(entry.typescriptRequiredDetections, entry.id).toEqual(counterpart.requiredDetections); + expect(entry.typescriptRequiredDetections, entry.id).toEqual(counterpart.typescript.requiredDetections); expect(entry.requiredDetections, entry.id).toEqual(expect.arrayContaining(["generated", "portable"])); expect(entry.requiredDetections.every(cohort => ["generated", "fixed", "portable"].includes(cohort)), entry.id).toBe(true); } @@ -156,3 +164,15 @@ describe("Rust fault catalog", () => { } }); }); + + +describe("Rust mutation build isolation", () => { + it("gives concurrent shards and partial runs separate Cargo target directories", () => { + const first = rustTargetDirectory("/repo", { shard: { index: 1, count: 6 } }); + const second = rustTargetDirectory("/repo", { shard: { index: 2, count: 6 } }); + const full = rustTargetDirectory("/repo", { shard: { index: 1, count: 1 } }); + const partial = rustTargetDirectory("/repo", { shard: { index: 1, count: 1 }, only: ["M01"] }); + expect(new Set([first, second, full, partial]).size).toBe(4); + expect(first).toBe("/repo/rust/target/semantic/shards/1-of-6"); + }); +}); From 107cf9dfe9b964d6ea02c2d8724917ba8d25c132 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Mon, 21 Sep 2026 18:06:07 -0700 Subject: [PATCH 26/31] fix(rust): close API and runtime review findings --- rust/Cargo.lock | 1 + rust/Cargo.toml | 1 + rust/README.md | 34 +++- rust/src/blocking.rs | 101 +++++++++++ rust/src/cpu_tests.rs | 336 +++++++++++++++++++++++++++++++++++ rust/src/engine.rs | 24 +-- rust/src/execution.rs | 99 ++++++++--- rust/src/flight.rs | 100 ++++++++++- rust/src/lib.rs | 3 + rust/src/observe.rs | 38 +++- rust/src/operation.rs | 28 ++- rust/src/runtime.rs | 14 ++ rust/src/shadow.rs | 60 ++++--- rust/src/testing.rs | 8 + rust/src/use_case.rs | 75 +++----- rust/tests/default_logger.rs | 42 +++++ rust/tests/formal/causal.rs | 11 ++ rust/tests/key_spec.rs | 51 ++++++ rust/tests/operation_api.rs | 201 +++++++++++++++++++++ 19 files changed, 1086 insertions(+), 141 deletions(-) create mode 100644 rust/src/blocking.rs create mode 100644 rust/src/cpu_tests.rs create mode 100644 rust/tests/default_logger.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index 6e9d6d18..be3c45d8 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -159,6 +159,7 @@ dependencies = [ "serde", "serde_json", "sha1", + "slab", "thiserror 2.0.20", "tokio", "zstd", diff --git a/rust/Cargo.toml b/rust/Cargo.toml index fafe1a58..80959b59 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -21,6 +21,7 @@ prometheus = ["dep:prometheus"] [dependencies] futures = "0.3" parking_lot = "0.12" +slab = "0.4" lru = "0.18" serde = { version = "1", features = ["derive"] } serde_json = { version = "1", features = ["float_roundtrip"] } diff --git a/rust/README.md b/rust/README.md index 12d002ad..62cf7405 100644 --- a/rust/README.md +++ b/rust/README.md @@ -49,7 +49,10 @@ async fn main() -> Result<(), BoxError> { A policy enables no cache layers by default. The example opts into request caching and a 30-second process-local cache; register the use case once at startup and create a scope for each request. `Arc` shares one cached value -without requiring `T: Clone`. +without requiring `T: Clone`. Settled memory entries holding another Rust type +are misses; compatible remote JSON may still decode into the requested type. +Simultaneous calls sharing a key also share one source result, so an incompatible +coalesced follower returns a type error. Use a consistent value type per key. Run the complete [basic example](./examples/basic.rs), which demonstrates a structured value, an asynchronous source and reuse across two request scopes: @@ -142,7 +145,19 @@ writes and surfaced invalidation errors. Writes are one native `SET`; invalidation dispatches `EVALSHA` and retries once with `EVAL`. No value write creates or extends a watermark. `DialCache::invalidate` affects shared remote authority; other processes' local entries and already acquired snapshots -retain the documented lifetime rules. +retain the documented lifetime rules. Inline operations with an explicit namespace +can invalidate that same entity through `invalidate_identity`: + +```rust,ignore +let identity = Identity::new("user", 42, "displayName") + .namespace("tenant-b").tracked(true); +// After committing the source mutation: +cache.invalidate_identity(identity, 0).await?; +``` + +An empty namespace inherits the cache instance's namespace. Invalidation covers +all tracked use cases and argument variants for that namespace, entity type and +ID; the identity's `tracked` flag does not restrict the maintenance operation. `Clock` separates wall time from elapsed time; `Runtime` supplies detached task admission and timers. The defaults are `SystemClock` (aligned to the @@ -171,7 +186,20 @@ sentinel decodes as JSON `null`, so `Option` destinations read it as `None`. Compression defaults to a 4,096-byte threshold and zstd level 3; `disable_compression` stores payloads raw while reads still accept compressed entries. The wire contract requires interoperable decompression, not identical -compressed bytes. +compressed bytes. The async engine dispatches payload compression at 64 KiB or +larger (and compression at levels 10–22 once the configured threshold is met) +and every zstd decompression through `Runtime::spawn_blocking`. Small raw +payloads remain inline; synchronous protocol helpers remain synchronous. + +The default CPU executor is shared across cache instances: two worker threads +and two queued jobs. Admission never waits for queue space. Saturation or worker +creation failure fails open: a read falls through to the source, and a failed +compression skips the write while preserving the source result. Custom runtimes +may supply their own bounded CPU executor. `StepRuntime` queues these jobs on its +controlled executor for deterministic tests. A shadow job retains its capacity +until its admitted CPU work finishes or is discarded, including after a deadline +or async runtime shutdown. Codecs supplied by the application still choose their +own scheduling; `FromSync` does not offload application serialization. `Observer` receives every public diagnostic as a typed `Event`. Shadow validation exists only to be observed, so a job is admitted only when the diff --git a/rust/src/blocking.rs b/rust/src/blocking.rs new file mode 100644 index 00000000..75490c4c --- /dev/null +++ b/rust/src/blocking.rs @@ -0,0 +1,101 @@ +//! Bounded CPU dispatch shared by production runtimes. + +use std::panic::{catch_unwind, AssertUnwindSafe}; +use std::sync::{mpsc, Arc, OnceLock}; + +use futures::channel::oneshot; +use parking_lot::Mutex; + +use crate::error::BoxError; +use crate::execution::PanicError; +use crate::flight::{panic_message, DROPPED_MESSAGE}; +use crate::runtime::Runtime; + +type Job = Box; + +struct Pool { + sender: mpsc::SyncSender, +} + +impl Pool { + fn new(workers: usize, queued: usize) -> std::io::Result { + let (sender, receiver) = mpsc::sync_channel::(queued); + let receiver = Arc::new(Mutex::new(receiver)); + for index in 0..workers { + let receiver = receiver.clone(); + std::thread::Builder::new() + .name(format!("dialcache-cpu-{index}")) + .spawn(move || loop { + let next = receiver.lock().recv(); + let Ok(job) = next else { break }; + // A custom job must not take a worker out of the pool. + let _ = catch_unwind(AssertUnwindSafe(job)); + })?; + } + Ok(Self { sender }) + } + + fn submit(&self, job: Job) -> Result<(), BoxError> { + self.sender + .try_send(job) + .map_err(|_| "DialCache CPU queue is full or unavailable".into()) + } +} + +pub(crate) fn submit(job: Job) -> Result<(), BoxError> { + static POOL: OnceLock> = OnceLock::new(); + POOL.get_or_init(|| Pool::new(2, 2).map_err(|e| e.to_string())) + .as_ref() + .map_err(|e| -> BoxError { format!("Could not start DialCache CPU workers: {e}").into() })? + .submit(job) +} + +/// Waiting does not own or cancel the admitted raw job. Its closure owns its +/// payload and any shadow slot until it finishes, even if the runtime closes. +pub(crate) async fn run( + runtime: &dyn Runtime, + job: impl FnOnce() -> Result + Send + 'static, +) -> Result { + let (send, receive) = oneshot::channel(); + catch_unwind(AssertUnwindSafe(|| { + runtime.spawn_blocking(Box::new(move || { + let result = catch_unwind(AssertUnwindSafe(job)) + .unwrap_or_else(|p| Err(Box::new(PanicError(panic_message(p))))); + let _ = send.send(result); + })) + })) + .map_err(|p| -> BoxError { Box::new(PanicError(panic_message(p))) })??; + receive + .await + .map_err(|_| -> BoxError { DROPPED_MESSAGE.into() })? +} + +#[cfg(test)] +mod tests { + use super::*; + use std::time::Duration; + + #[test] + fn pool_bounds_admission_and_keeps_worker_after_panic() { + let pool = Pool::new(1, 1).unwrap(); + let (started, running) = mpsc::channel(); + let (release, gate) = mpsc::channel(); + pool.submit(Box::new(move || { + started.send(()).unwrap(); + gate.recv_timeout(Duration::from_secs(5)).unwrap(); + panic!("worker must survive"); + })) + .unwrap(); + running.recv_timeout(Duration::from_secs(5)).unwrap(); + let (finished, done) = mpsc::channel(); + pool.submit(Box::new(move || { + finished.send(()).unwrap(); + })) + .unwrap(); + assert!(pool + .submit(Box::new(|| panic!("rejected task ran"))) + .is_err()); + release.send(()).unwrap(); + done.recv_timeout(Duration::from_secs(5)).unwrap(); + } +} diff --git a/rust/src/cpu_tests.rs b/rust/src/cpu_tests.rs new file mode 100644 index 00000000..680db718 --- /dev/null +++ b/rust/src/cpu_tests.rs @@ -0,0 +1,336 @@ +//! Native scheduling and ownership tests for built-in CPU work. +use std::collections::VecDeque; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use futures::future::BoxFuture; +use parking_lot::Mutex; + +use crate::observe::ShadowOutcome; +use crate::protocol::{compress_payload, CompressionConfig}; +use crate::testing::{StepRuntime, TestExecutor, WALL_EPOCH_MS}; +use crate::*; + +type Job = Box; +struct CpuRuntime { + step: Arc, + jobs: Mutex>, + reject: AtomicBool, +} +impl Runtime for CpuRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.step.spawn(task); + } + fn defer(&self, task: BoxFuture<'static, ()>) { + self.step.defer(task); + } + fn sleep(&self, delay: Duration) -> BoxFuture<'static, ()> { + self.step.sleep(delay) + } + fn spawn_blocking(&self, task: Job) -> Result<(), BoxError> { + if self.reject.load(Ordering::SeqCst) { + return Err("test CPU rejection".into()); + } + self.jobs.lock().push_back(task); + Ok(()) + } +} +impl CpuRuntime { + fn take(&self) -> Job { + self.jobs.lock().pop_front().expect("CPU work admitted") + } +} +struct SnapshotRemote { + result: ReadResult, + writes: AtomicUsize, +} +impl Remote for SnapshotRemote { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(std::future::ready(Ok(self.result.clone()))) + } + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + self.writes.fetch_add(1, Ordering::SeqCst); + Box::pin(std::future::ready(Ok(()))) + } + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(std::future::ready(Ok(()))) + } +} +#[derive(Default)] +struct Events(Mutex>); +impl Observer for Events { + fn observes_shadow_outcomes(&self) -> bool { + true + } + fn observe(&self, event: &Event) { + if let Event::ShadowValidation { outcome, .. } = event { + self.0.lock().push(*outcome); + } + } +} +struct TextCodec(Arc); +impl SyncCodec for TextCodec { + fn encode(&self, text: &String) -> Result { + Ok(Payload::text(text.clone())) + } + fn decode(&self, payload: Payload) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(payload.as_text().into_owned()) + } +} +fn text() -> String { + "a".repeat(128 * 1024) +} +fn snapshot(decode: bool) -> ReadResult { + if decode { + ReadResult::Hit(Frame { + created_at_ms: WALL_EPOCH_MS as u64, + payload: compress_payload( + Payload::text(text()), + &CompressionConfig::default(), + limits::MAX_DECOMPRESSED_BYTES, + ) + .unwrap() + .payload, + }) + } else { + ReadResult::miss(MissReason::ValueAbsent) + } +} +fn operation(decodes: &Arc, shadow: bool) -> Operation { + let mut policy = Policy::default().remote_ttl_sec(60); + if shadow { + policy = policy.remote_ramp(0.0).shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: None, + }); + } + Operation::with_codec( + Identity::new("thing", "one", "CPU"), + Arc::new(FromSync(TextCodec(decodes.clone()))), + |a, b| a == b, + ) + .policy(policy) + .budget(SourceBudget::Millis(5)) +} +fn setup( + decode: bool, +) -> ( + TestExecutor, + DialCache, + Arc, + Arc, + Arc, +) { + let executor = TestExecutor::new(WALL_EPOCH_MS); + let runtime = Arc::new(CpuRuntime { + step: executor.runtime.clone(), + jobs: Mutex::new(VecDeque::new()), + reject: AtomicBool::new(false), + }); + let remote = Arc::new(SnapshotRemote { + result: snapshot(decode), + writes: AtomicUsize::new(0), + }); + let events = Arc::new(Events::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(runtime.clone()) + .remote_arc(remote.clone()) + .observer_arc(events.clone()) + .build() + .unwrap(); + (executor, cache, runtime, remote, events) +} +fn invoke(executor: &mut TestExecutor, cache: &DialCache, op: Operation) -> Arc { + let cache = cache.clone(); + executor.block_on(async move { + let scope = cache.enable_guard(); + cache + .get_or_load(scope.scope(), op, |_| async { Ok(text()) }) + .await + .unwrap() + }) +} + +#[test] +fn rejected_cpu_work_fails_open_through_the_cache_api() { + for decode in [false, true] { + let (mut executor, cache, runtime, remote, events) = setup(decode); + runtime.reject.store(true, Ordering::SeqCst); + let decodes = Arc::new(AtomicUsize::new(0)); + for shadow in [false, true] { + for _ in 0..2 { + assert_eq!( + *invoke(&mut executor, &cache, operation(&decodes, shadow)), + text() + ); + } + assert!(cache.core.state.lock().flights.is_empty()); + assert!(cache.core.state.lock().shadows.is_empty()); + } + assert_eq!(remote.writes.load(Ordering::SeqCst), 0); + assert_eq!(decodes.load(Ordering::SeqCst), 0); + assert_eq!( + *events.0.lock(), + vec![ + if decode { + ShadowOutcome::DeserializationError + } else { + ShadowOutcome::FillError + }; + 2 + ] + ); + } +} + +#[test] +fn shadow_deadline_keeps_cpu_ownership_and_prevents_subsequent_phases() { + for decode in [false, true] { + for deliver in [false, true] { + let (mut executor, cache, runtime, remote, events) = setup(decode); + let decodes = Arc::new(AtomicUsize::new(0)); + invoke(&mut executor, &cache, operation(&decodes, true)); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + assert_eq!(runtime.jobs.lock().len(), 1); + executor.advance(5, deliver); + assert_eq!( + cache.core.state.lock().shadows.len(), + 1, + "timeout released raw CPU ownership" + ); + runtime.take()(); + executor.drain(); + assert_eq!(*events.0.lock(), vec![ShadowOutcome::Timeout]); + assert_eq!( + decodes.load(Ordering::SeqCst), + 0, + "codec ran after decode deadline" + ); + assert_eq!( + remote.writes.load(Ordering::SeqCst), + 0, + "write ran after compression deadline" + ); + assert!(cache.core.state.lock().shadows.is_empty()); + } + } +} + +#[test] +fn queued_cpu_job_keeps_shadow_slot_after_executor_shutdown() { + for discard in [false, true] { + let (mut executor, cache, runtime, _, _) = setup(false); + invoke( + &mut executor, + &cache, + operation(&Arc::new(AtomicUsize::new(0)), true), + ); + drop(executor); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + let job = runtime.take(); + if discard { + drop(job); + } else { + job(); + } + assert!(cache.core.state.lock().shadows.is_empty()); + } +} + +#[test] +fn running_cpu_job_keeps_shadow_slot_after_executor_shutdown() { + let (mut executor, cache, runtime, _, _) = setup(false); + invoke( + &mut executor, + &cache, + operation(&Arc::new(AtomicUsize::new(0)), true), + ); + let job = runtime.take(); + let (started, running) = std::sync::mpsc::channel(); + let (release, gate) = std::sync::mpsc::channel(); + let worker = std::thread::spawn(move || { + started.send(()).unwrap(); + gate.recv_timeout(Duration::from_secs(5)).unwrap(); + job(); + }); + running.recv_timeout(Duration::from_secs(5)).unwrap(); + drop(executor); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + release.send(()).unwrap(); + worker.join().unwrap(); + assert!(cache.core.state.lock().shadows.is_empty()); +} + +#[cfg(feature = "tokio")] +#[tokio::test(flavor = "current_thread")] +async fn large_codec_phases_allow_an_independent_short_timer_to_progress() { + struct GatedRuntime { + inner: TokioRuntime, + started: tokio::sync::mpsc::UnboundedSender, + release: Arc>>, + } + impl Runtime for GatedRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.inner.spawn(task); + } + fn sleep(&self, delay: Duration) -> BoxFuture<'static, ()> { + self.inner.sleep(delay) + } + fn spawn_blocking(&self, task: Job) -> Result<(), BoxError> { + let started = self.started.clone(); + let release = self.release.clone(); + self.inner.spawn_blocking(Box::new(move || { + started.send(std::thread::current().id()).unwrap(); + release.lock().recv_timeout(Duration::from_secs(5)).unwrap(); + task(); + })) + } + } + for decode in [false, true] { + let (started, mut started_rx) = tokio::sync::mpsc::unbounded_channel(); + let (release, gate) = std::sync::mpsc::channel(); + let runtime = GatedRuntime { + inner: TokioRuntime::current().unwrap(), + started, + release: Arc::new(Mutex::new(gate)), + }; + let remote = Arc::new(SnapshotRemote { + result: snapshot(decode), + writes: AtomicUsize::new(0), + }); + let cache = DialCache::builder() + .clock_arc(crate::testing::VirtualClock::new(WALL_EPOCH_MS)) + .runtime(runtime) + .remote_arc(remote) + .build() + .unwrap(); + let decodes = Arc::new(AtomicUsize::new(0)); + let op = operation(&decodes, false); + let pending = tokio::spawn(async move { + let guard = cache.enable_guard(); + cache + .get_or_load(guard.scope(), op, |_| async { Ok(text()) }) + .await + .unwrap() + }); + let worker = tokio::time::timeout(Duration::from_secs(5), started_rx.recv()) + .await + .unwrap() + .unwrap(); + assert_ne!(worker, std::thread::current().id()); + tokio::time::sleep(Duration::from_millis(2)).await; + assert!(!pending.is_finished(), "the CPU gate should still be held"); + release.send(()).unwrap(); + assert_eq!( + *tokio::time::timeout(Duration::from_secs(5), pending) + .await + .unwrap() + .unwrap(), + text() + ); + assert_eq!(decodes.load(Ordering::SeqCst), usize::from(decode)); + } +} diff --git a/rust/src/engine.rs b/rust/src/engine.rs index e3defbb7..c7b6dd43 100644 --- a/rust/src/engine.rs +++ b/rust/src/engine.rs @@ -595,7 +595,15 @@ impl DialCache { self.invalidate_identity(identity, future_buffer_ms).await } - async fn invalidate_identity( + /// Advance the watermark for the entity named by `identity` after its + /// source mutation commits. An empty namespace inherits this instance's; + /// an explicit namespace is preserved, exactly as in `get_or_load`. + /// + /// All use cases and argument variants share the entity's watermark. + /// `tracked` is ignored: this operation always invalidates tracked remote + /// values. Local entries and already acquired snapshots retain their lifetimes. + /// Requires a remote adapter; mutation failures are returned to the caller. + pub async fn invalidate_identity( &self, mut identity: Identity, future_buffer_ms: u64, @@ -685,16 +693,11 @@ impl DialCache { F: Fn(Scope) -> Fut + Send + Sync + 'static, Fut: Future> + Send + 'static, { - let parts = operation.erase(); + let (identity, metadata) = operation.erase(); let erased = ErasedOperation { - identity: parts.identity, + identity, identity_provider: None, - policy: parts.policy, - budget: parts.budget, - codec: parts.codec, - compare: parts.compare, - should_recover: parts.should_recover, - preview: parts.preview, + metadata, load: erase_load(load), }; let value = self.execute(scope, erased).await?; @@ -740,10 +743,11 @@ impl DialCache { pub(crate) fn validate_operation(operation: &ErasedOperation) -> Result<(), Error> { operation + .metadata .policy .validate() .map_err(|e| ConfigError::invalid(e.to_string()))?; - if let crate::operation::SourceBudget::Millis(ms) = operation.budget { + if let crate::operation::SourceBudget::Millis(ms) = operation.metadata.budget { if !(1..=MAX_DEADLINE_MS).contains(&ms) { return Err(ConfigError::invalid(format!( "DialCache source budget must be a positive integer no greater than {MAX_DEADLINE_MS} ms" diff --git a/rust/src/execution.rs b/rust/src/execution.rs index fb71f028..c86ce9a9 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -29,6 +29,7 @@ use crate::protocol::{ }; use crate::remote::{Frame, ReadContext, ReadRequest, ReadResult, WriteRequest}; use crate::scope::{Owner, Scope}; +use crate::shadow::ShadowWork; /// The raw outcome of one adapter read. pub(crate) type RawRead = Result; @@ -204,7 +205,7 @@ pub(crate) async fn run(core: Arc, scope: Scope, op: Arc) }; let resolved = overlay.and_then(|overlay| { resolve_policy( - &op.policy, + &op.metadata.policy, overlay.as_ref(), &keys.logical, PolicyDefaults { @@ -327,7 +328,7 @@ async fn source_with_budget( ) -> ValueResult { let clock = core.clock.clone(); let start = clock.elapsed(); - let budget = op.budget.millis(); + let budget = op.metadata.budget.millis(); let pending: Settled = start_pending( core.runtime.as_ref(), { @@ -423,18 +424,22 @@ impl Execution { labels: self.labels(Layer::RequestLocal), seconds: self.seconds_since(start), }); - if let (true, Some(value)) = (live, memo) { - return Ok(value); - } + let reason = match (live, memo) { + (true, Some(value)) if value.as_ref().type_id() == self.op.metadata.value_type => { + return Ok(value); + } + (true, Some(_)) => MissReason::Unclassified, + _ => MissReason::ValueAbsent, + }; self.emit(Event::Miss { labels: self.labels(Layer::RequestLocal), - reason: MissReason::ValueAbsent, + reason, }); let result = self.clone().shared(Layer::RequestLocal).await; if let Ok(value) = &result { let displaced = { let mut state = owner.state.lock(); - if state.live { + if state.live && value.as_ref().type_id() == self.op.metadata.value_type { state.memo.insert(self.keys.logical.clone(), value.clone()) } else { None @@ -578,13 +583,17 @@ impl Execution { labels: self.labels(Layer::Local), seconds: self.seconds_since(start), }); - if let Some(value) = found { - return Ok(value); - } + let reason = match found { + Some(value) if value.as_ref().type_id() == self.op.metadata.value_type => { + return Ok(value); + } + Some(_) => MissReason::Unclassified, + None => MissReason::ValueAbsent, + }; local_miss = true; self.emit(Event::Miss { labels: self.labels(Layer::Local), - reason: MissReason::ValueAbsent, + reason, }); } } @@ -801,7 +810,7 @@ impl Execution { }); RemoteValue::Retained { frame } } else { - match self.decode(&frame, Layer::Remote).await { + match self.decode(&frame, Layer::Remote, None).await { Ok(value) => RemoteValue::Hit { value, frame }, Err(_) => { self.emit(Event::Miss { @@ -827,9 +836,29 @@ impl Execution { &self, frame: &Frame, layer: Layer, + shadow: Option<&ShadowWork>, ) -> Result { let decompress_started = self.elapsed(); - let expanded = decompress_payload(frame.payload.clone(), MAX_DECOMPRESSED_BYTES); + let payload = frame.payload.clone(); + let expanded = if payload.binary && matches!(payload.bytes.first(), Some(1 | 2)) { + let shadow = shadow.cloned(); + match crate::blocking::run(self.core.runtime.as_ref(), move || { + if shadow.as_ref().is_some_and(ShadowWork::expired) { + return Err("shadow deadline elapsed before decompression".into()); + } + Ok(decompress_payload(payload, MAX_DECOMPRESSED_BYTES)) + }) + .await + { + Ok(expanded) => expanded, + Err(error) => { + self.error_event(layer, ErrorKind::Compression, false); + return Err(error); + } + } + } else { + decompress_payload(payload, MAX_DECOMPRESSED_BYTES) + }; if let Some(outcome) = expanded.outcome { self.emit(Event::Compression { labels: self.labels(layer), @@ -841,8 +870,11 @@ impl Execution { seconds: self.seconds_since(decompress_started), }); } + if shadow.is_some_and(ShadowWork::expired) { + return Err("shadow deadline elapsed after decompression".into()); + } let start = self.elapsed(); - let codec = self.op.codec.clone(); + let codec = self.op.metadata.codec.clone(); let decoded = match AssertUnwindSafe(async { codec.decode(expanded.payload).await }) .catch_unwind() .await @@ -871,13 +903,13 @@ impl Execution { } /// Prepare and dispatch one remote write. `Ok(false)` means the write was - /// fenced or refused by `allowed`; `Ok(true)` means it was dispatched. + /// fenced or its shadow deadline expired; `Ok(true)` means it was dispatched. pub(crate) async fn put_remote( &self, value: &StoredValue, fence: Option, layer: Layer, - allowed: Option<&(dyn Fn() -> bool + Send + Sync)>, + shadow: Option<&ShadowWork>, ) -> Result { let fence = if self.identity.tracked { fence } else { None }; if let Some(fence) = fence { @@ -887,7 +919,7 @@ impl Execution { } } let start = self.elapsed(); - let codec = self.op.codec.clone(); + let codec = self.op.metadata.codec.clone(); let encoded = match AssertUnwindSafe(async { codec.encode(value).await }) .catch_unwind() .await @@ -904,6 +936,9 @@ impl Execution { seconds: self.seconds_since(start), }); let mut payload = encoded?; + if shadow.is_some_and(ShadowWork::expired) { + return Ok(false); + } self.emit(Event::Size { labels: self.labels(layer), bytes: payload.len() as u64, @@ -911,11 +946,28 @@ impl Execution { match &self.core.compression { Some(config) => { let compress_started = self.elapsed(); - let compressed = match compress_payload(payload, config, MAX_DECOMPRESSED_BYTES) { + let config = *config; + let offload = payload.len() >= 64 * 1024 + || (config.level >= 10 && payload.len() >= config.threshold_bytes); + let result = if offload { + let shadow = shadow.cloned(); + crate::blocking::run(self.core.runtime.as_ref(), move || { + if shadow.as_ref().is_some_and(ShadowWork::expired) { + return Err("shadow deadline elapsed before compression".into()); + } + compress_payload(payload, &config, MAX_DECOMPRESSED_BYTES) + .map_err(|e| Box::new(e) as BoxError) + }) + .await + } else { + compress_payload(payload, &config, MAX_DECOMPRESSED_BYTES) + .map_err(|e| Box::new(e) as BoxError) + }; + let compressed = match result { Ok(compressed) => compressed, Err(error) => { self.error_event(layer, ErrorKind::Compression, false); - return Err(Box::new(error)); + return Err(error); } }; payload = compressed.payload; @@ -947,10 +999,8 @@ impl Execution { labels: self.labels(layer), bytes: payload.len() as u64, }); - if let Some(allowed) = allowed { - if !allowed() { - return Ok(false); - } + if shadow.is_some_and(ShadowWork::expired) { + return Ok(false); } let stamp = self.write_timestamp(layer)?; if let Some(fence) = fence { @@ -1002,6 +1052,7 @@ impl Execution { pub(crate) fn can_recover(&self, error: &Error) -> bool { let predicate = self .op + .metadata .should_recover .clone() .or_else(|| self.core.should_recover.clone()); @@ -1046,7 +1097,7 @@ impl Execution { self.recovery_event(RecoveryOutcome::Miss, None); return None; } - let value = match self.decode(frame, Layer::Remote).await { + let value = match self.decode(frame, Layer::Remote, None).await { Ok(value) => value, Err(error) => { self.core.log(LogEvent::RecoveryDecodeFailed(error)); diff --git a/rust/src/flight.rs b/rust/src/flight.rs index 93db895d..60e723a8 100644 --- a/rust/src/flight.rs +++ b/rust/src/flight.rs @@ -10,6 +10,7 @@ use std::time::Duration; use futures::FutureExt; use parking_lot::Mutex; +use slab::Slab; use crate::error::Error; use crate::local::StoredValue; @@ -17,7 +18,7 @@ use crate::runtime::Runtime; struct SettledState { value: Option, - wakers: Vec, + wakers: Slab, } /// A value that is set at most once and observed by any number of waiters. @@ -38,7 +39,7 @@ impl Settled { Settled { inner: Arc::new(Mutex::new(SettledState { value: None, - wakers: Vec::new(), + wakers: Slab::new(), })), } } @@ -53,7 +54,7 @@ impl Settled { state.value = Some(value); std::mem::take(&mut state.wakers) }; - for waker in wakers { + for (_, waker) in wakers { waker.wake(); } true @@ -67,29 +68,57 @@ impl Settled { pub(crate) fn wait(&self) -> Wait { Wait { settled: self.clone(), + registration: None, } } } pub(crate) struct Wait { settled: Settled, + registration: Option, } impl Future for Wait { type Output = T; fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let mut state = self.settled.inner.lock(); + let this = self.get_mut(); + let mut state = this.settled.inner.lock(); if let Some(value) = &state.value { return Poll::Ready(value.clone()); } - if !state.wakers.iter().any(|w| w.will_wake(cx.waker())) { - state.wakers.push(cx.waker().clone()); - } + let replaced = match this.registration { + Some(slot) => Some(std::mem::replace( + &mut state.wakers[slot], + cx.waker().clone(), + )), + None => { + this.registration = Some(state.wakers.insert(cx.waker().clone())); + None + } + }; + drop(state); + drop(replaced); Poll::Pending } } +impl Drop for Wait { + fn drop(&mut self) { + let removed = { + let mut state = self.settled.inner.lock(); + if state.value.is_none() { + self.registration + .take() + .map(|slot| state.wakers.remove(slot)) + } else { + None + } + }; + drop(removed); + } +} + impl Unpin for Wait {} /// Describe a panic payload. @@ -191,3 +220,60 @@ impl Flight { self.followers.load(Ordering::Relaxed) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::AtomicUsize; + use std::task::Wake; + + #[derive(Default)] + struct Counter(AtomicUsize); + impl Wake for Counter { + fn wake(self: Arc) { + self.0.fetch_add(1, Ordering::SeqCst); + } + } + + #[test] + fn many_waiters_replace_and_remove_their_own_registration() { + let cell = Settled::new(); + let mut waiters: Vec<_> = (0..10_000).map(|_| cell.wait()).collect(); + let counters: Vec<_> = (0..10_000).map(|_| Arc::new(Counter::default())).collect(); + for (wait, counter) in waiters.iter_mut().zip(&counters) { + let waker = Waker::from(counter.clone()); + for _ in 0..3 { + assert!(Pin::new(&mut *wait) + .poll(&mut Context::from_waker(&waker)) + .is_pending()); + } + } + let replacement = Arc::new(Counter::default()); + let waker = Waker::from(replacement.clone()); + assert!(Pin::new(&mut waiters[0]) + .poll(&mut Context::from_waker(&waker)) + .is_pending()); + // Cancellation frees slots; the surviving waiter must not remove a reused slot. + waiters.truncate(5_000); + let mut later = cell.wait(); + assert!(Pin::new(&mut later) + .poll(&mut Context::from_waker(&waker)) + .is_pending()); + assert!(cell.settle(7)); + assert!(!cell.settle(8)); + assert_eq!(replacement.0.load(Ordering::SeqCst), 2); + for (i, counter) in counters.iter().enumerate() { + assert_eq!( + counter.0.load(Ordering::SeqCst), + usize::from(i > 0 && i < 5_000) + ); + } + for mut wait in waiters { + assert_eq!( + Pin::new(&mut wait).poll(&mut Context::from_waker(&waker)), + Poll::Ready(7) + ); + } + drop(later); // Settlement removed registrations; stale indices must be ignored. + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index ecbfe6a4..2a4d8ed1 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -10,9 +10,12 @@ #![warn(missing_debug_implementations)] #![warn(missing_docs)] +mod blocking; pub mod cancel; pub mod clock; pub mod codec; +#[cfg(test)] +mod cpu_tests; pub mod datadog; mod deadline; mod engine; diff --git a/rust/src/observe.rs b/rust/src/observe.rs index b958e54c..9364b0c8 100644 --- a/rust/src/observe.rs +++ b/rust/src/observe.rs @@ -502,23 +502,43 @@ impl LogEvent { impl fmt::Display for LogEvent { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - LogEvent::KeyConstructionFailed(e) => write!(f, "Could not construct DialCache key: {e}"), - LogEvent::PolicyResolutionFailed(e) => write!(f, "Could not resolve DialCache key config: {e}"), + LogEvent::KeyConstructionFailed(e) => { + write!(f, "Could not construct DialCache key: {e}") + } + LogEvent::PolicyResolutionFailed(e) => { + write!(f, "Could not resolve DialCache key config: {e}") + } LogEvent::LocalReadFailed(e) => write!(f, "Error getting value from local cache: {e}"), LogEvent::LocalWriteFailed(e) => write!(f, "Error putting value in local cache: {e}"), LogEvent::RemoteReadFailed(e) => write!(f, "Error getting value from Redis cache: {e}"), LogEvent::RemoteWriteFailed(e) => write!(f, "Error putting value in Redis cache: {e}"), LogEvent::RecoveryPredicateFailed(e) => { - write!(f, "DialCache stale recovery predicate failed; recovery was denied: {e}") + write!( + f, + "DialCache stale recovery predicate failed; recovery was denied: {e}" + ) } - LogEvent::RecoveryDecodeFailed(e) => write!(f, "Error using retained Redis value during stale recovery: {e}"), - LogEvent::ShadowFillFailed(e) => write!(f, "Error populating Redis from DialCache shadow work: {e}"), - LogEvent::ShadowMismatch(d) => write!( + LogEvent::RecoveryDecodeFailed(e) => write!( f, - "DialCache shadow validation mismatch: namespace={} useCase={} keyType={} cacheKey={}", - d.namespace, d.use_case, d.key_type, d.cache_key + "Error using retained Redis value during stale recovery: {e}" ), - LogEvent::InvalidationFailed(e) => write!(f, "Error writing DialCache invalidation watermark: {e}"), + LogEvent::ShadowFillFailed(e) => { + write!(f, "Error populating Redis from DialCache shadow work: {e}") + } + LogEvent::ShadowMismatch(d) => { + write!(f, "DialCache shadow validation mismatch: namespace={} useCase={} keyType={} cacheKey={}", + d.namespace, d.use_case, d.key_type, d.cache_key)?; + if let Some(value) = &d.cached_value_json { + write!(f, " cachedValue={value}")?; + } + if let Some(value) = &d.source_value_json { + write!(f, " sourceValue={value}")?; + } + Ok(()) + } + LogEvent::InvalidationFailed(e) => { + write!(f, "Error writing DialCache invalidation watermark: {e}") + } } } } diff --git a/rust/src/operation.rs b/rust/src/operation.rs index a0cc732d..615dc025 100644 --- a/rust/src/operation.rs +++ b/rust/src/operation.rs @@ -1,6 +1,6 @@ //! Operations: the typed description of one cached call and its erased form. -use std::any::Any; +use std::any::TypeId; use std::sync::Arc; use futures::future::BoxFuture; @@ -48,7 +48,7 @@ pub type Preview = Arc Option + Send + Sync>; /// One inline cached call: its identity, static policy, budget and codecs. /// -/// Registered use cases build this for every call; use it directly with +/// Use it directly with /// [`DialCache::get_or_load`](crate::DialCache::get_or_load) when a stable use /// case is declared at the call site instead of being registered. pub struct Operation { @@ -159,12 +159,12 @@ impl Operation { self } - pub(crate) fn erase(self) -> ErasedParts { + pub(crate) fn erase(self) -> (Identity, Arc) { let codec = self.codec; let comparator = self.comparator; let preview = self.preview; - ErasedParts { - identity: self.identity, + let metadata = OperationMetadata { + value_type: TypeId::of::(), policy: self.policy, budget: self.budget, codec: Arc::new(TypedCodec { codec }), @@ -180,7 +180,8 @@ impl Operation { value.downcast_ref::().and_then(|v| preview(v)) }) as Arc Option + Send + Sync> }), - } + }; + (self.identity, Arc::new(metadata)) } } @@ -192,8 +193,9 @@ pub(crate) type ErasedCompare = pub(crate) type ErasedPreview = Arc Option + Send + Sync>; pub(crate) type IdentityProvider = Arc Result + Send + Sync>; -pub(crate) struct ErasedParts { - pub identity: Identity, +/// Immutable adapters and policy, shared by every registered invocation. +pub(crate) struct OperationMetadata { + pub value_type: TypeId, pub policy: Policy, pub budget: SourceBudget, pub codec: Arc, @@ -206,12 +208,7 @@ pub(crate) struct ErasedParts { pub(crate) struct ErasedOperation { pub identity: Identity, pub identity_provider: Option, - pub policy: Policy, - pub budget: SourceBudget, - pub codec: Arc, - pub compare: ErasedCompare, - pub should_recover: Option, - pub preview: Option, + pub metadata: Arc, pub load: ErasedLoad, } @@ -265,6 +262,3 @@ pub(crate) fn downcast_value( ))) }) } - -#[allow(dead_code)] -fn _assert_any(_: &dyn Any) {} diff --git a/rust/src/runtime.rs b/rust/src/runtime.rs index 5a6bde2f..a3ad71b0 100644 --- a/rust/src/runtime.rs +++ b/rust/src/runtime.rs @@ -19,6 +19,20 @@ pub trait Runtime: Send + Sync + 'static { fn defer(&self, task: BoxFuture<'static, ()>) { self.spawn(task) } + /// Admit synchronous CPU work without blocking the async executor. + /// + /// The default uses a process-wide pool of two threads and two queued jobs. + /// A full queue or thread-creation failure returns an error; cache plumbing + /// fails open. Once admitted, a job owns its resources until execution or + /// destruction, independently of the future waiting for its result. + /// Custom runtimes may override this with their own bounded CPU executor. + /// A controlled test runtime may queue the job for deterministic execution. + fn spawn_blocking( + &self, + task: Box, + ) -> Result<(), crate::BoxError> { + crate::blocking::submit(task) + } /// A timer that completes once `duration` has passed. fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()>; } diff --git a/rust/src/shadow.rs b/rust/src/shadow.rs index a5036b59..ea4d0a64 100644 --- a/rust/src/shadow.rs +++ b/rust/src/shadow.rs @@ -68,6 +68,25 @@ impl ShadowFlight { } } +/// The deadline and owned capacity of one shadow job. CPU closures retain a +/// clone so runtime teardown cannot release capacity while raw work continues. +#[derive(Clone)] +pub(crate) struct ShadowWork { + flight: Arc, + slot: Arc, + started: Duration, + budget: Duration, +} + +impl ShadowWork { + pub(crate) fn expired(&self) -> bool { + if since(self.slot.core.clock.as_ref(), self.started) >= self.budget { + self.flight.abandon(); + } + self.flight.is_abandoned() + } +} + #[derive(Clone)] struct Verdict { outcome: ShadowOutcome, @@ -171,7 +190,7 @@ impl Execution { } if verdict.outcome == ShadowOutcome::Mismatch && self.policy.shadow.log_mismatches { let preview = |value: &StoredValue| -> Option { - let preview = self.op.preview.as_ref()?; + let preview = self.op.metadata.preview.as_ref()?; catch_unwind(AssertUnwindSafe(|| preview(value))) .ok() .flatten() @@ -209,23 +228,28 @@ impl Execution { }; let budget_ms = self .op + .metadata .budget .millis() .unwrap_or(DEFAULT_FALLBACK_TIMEOUT_MS); + let slot = Arc::new(slot); + let work = ShadowWork { + flight: flight.clone(), + slot: slot.clone(), + started, + budget: Duration::from_millis(budget_ms), + }; let reads: Arc>>> = Arc::new(Mutex::new(Vec::new())); let validation: Settled = start_pending( self.core.runtime.as_ref(), { let x = self.clone(); - let flight = flight.clone(); let reads = reads.clone(); async move { let verdict = match AssertUnwindSafe(x.clone().validate( - flight.clone(), + work, frame, source, - started, - budget_ms, reads.clone(), )) .catch_unwind() @@ -319,21 +343,12 @@ impl Execution { async fn validate( self: Arc, - flight: Arc, + work: ShadowWork, mut frame: Option, source: Option>, - started: Duration, - budget_ms: u64, reads: Arc>>>, ) -> Verdict { - let clock = self.core.clock.clone(); - let budget = Duration::from_millis(budget_ms); - let expired = || { - if since(clock.as_ref(), started) >= budget { - flight.abandon(); - } - flight.is_abandoned() - }; + let expired = || work.expired(); if expired() { return Verdict::of(ShadowOutcome::Timeout); } @@ -360,7 +375,7 @@ impl Execution { } let value: ValueResult = match &source { Some(source) => { - let result = match select(source.wait(), flight.stop.wait()).await { + let result = match select(source.wait(), work.flight.stop.wait()).await { Either::Left((result, _)) => result, Either::Right(((), _)) => return Verdict::of(ShadowOutcome::Timeout), }; @@ -386,9 +401,8 @@ impl Execution { return Verdict::of(ShadowOutcome::Timeout); } if fill { - let allowed = || !expired(); let filled = self - .put_remote(&value, fence, Layer::RemoteShadow, Some(&allowed)) + .put_remote(&value, fence, Layer::RemoteShadow, Some(&work)) .await; if expired() { return Verdict::of(ShadowOutcome::Timeout); @@ -405,14 +419,18 @@ impl Execution { let Some(frame) = frame else { return Verdict::of(ShadowOutcome::Timeout); }; - let cached = match self.decode(&frame, Layer::RemoteShadow).await { + let decoded = self.decode(&frame, Layer::RemoteShadow, Some(&work)).await; + if expired() { + return Verdict::of(ShadowOutcome::Timeout); + } + let cached = match decoded { Ok(cached) => cached, Err(_) => return Verdict::of(ShadowOutcome::DeserializationError), }; if expired() { return Verdict::of(ShadowOutcome::Timeout); } - let compare = self.op.compare.clone(); + let compare = self.op.metadata.compare.clone(); let matches = match catch_unwind(AssertUnwindSafe(|| compare(&cached, &value))) { Ok(Ok(matches)) => matches, _ => return Verdict::of(ShadowOutcome::ComparisonError), diff --git a/rust/src/testing.rs b/rust/src/testing.rs index ec0d742e..2f18cc35 100644 --- a/rust/src/testing.rs +++ b/rust/src/testing.rs @@ -237,6 +237,14 @@ impl Runtime for StepRuntime { self.queues.lock().deferred.push(task); } + fn spawn_blocking( + &self, + task: Box, + ) -> Result<(), crate::BoxError> { + self.spawn(Box::pin(async move { task() })); + Ok(()) + } + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { self.clock.sleep(duration) } diff --git a/rust/src/use_case.rs b/rust/src/use_case.rs index 30c82c11..b41613e0 100644 --- a/rust/src/use_case.rs +++ b/rust/src/use_case.rs @@ -13,8 +13,8 @@ use crate::engine::DialCache; use crate::error::{BoxError, ConfigError, Error}; use crate::identity::{normalize_args, ArgValue, Identity, IntoKeyId}; use crate::operation::{ - downcast_value, erase_load, Comparator, ErasedOperation, Operation, Preview, RecoveryPredicate, - SourceBudget, + downcast_value, erase_load, Comparator, ErasedOperation, Operation, OperationMetadata, Preview, + RecoveryPredicate, SourceBudget, }; use crate::policy::Policy; use crate::scope::Scope; @@ -277,18 +277,21 @@ where } } self.cache.register_use_case(&self.use_case)?; + let (identity, metadata) = Operation { + identity: Identity::new(self.key_type, "", self.use_case).tracked(self.tracked), + policy: self.policy, + budget: self.budget, + codec, + comparator, + should_recover: self.should_recover, + preview, + } + .erase(); Ok(UseCase { inner: Arc::new(UseCaseInner { cache: self.cache, - key_type: self.key_type, - use_case: self.use_case, - tracked: self.tracked, - policy: self.policy, - budget: self.budget, - codec, - comparator, - should_recover: self.should_recover, - preview, + identity, + metadata, key, source, }), @@ -335,15 +338,8 @@ where struct UseCaseInner { cache: DialCache, - key_type: String, - use_case: String, - tracked: bool, - policy: Policy, - budget: SourceBudget, - codec: Arc>, - comparator: Comparator, - should_recover: Option, - preview: Option>, + identity: Identity, + metadata: Arc, key: KeyFn, source: SourceFn, } @@ -368,8 +364,8 @@ impl Clone for UseCase { impl std::fmt::Debug for UseCase { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("UseCase") - .field("key_type", &self.inner.key_type) - .field("use_case", &self.inner.use_case) + .field("key_type", &self.inner.identity.key_type) + .field("use_case", &self.inner.identity.use_case) .finish() } } @@ -381,35 +377,19 @@ where { /// The operation name given at registration. pub fn use_case(&self) -> &str { - &self.inner.use_case + &self.inner.identity.use_case } /// The entity type given at registration. pub fn key_type(&self) -> &str { - &self.inner.key_type + &self.inner.identity.key_type } /// Get the value for `args` through the configured layers. pub async fn get(&self, scope: &Scope, args: Args) -> Result, Error> { let inner = self.inner.clone(); - let identity = Identity { - namespace: inner.cache.core.namespace.to_string(), - key_type: inner.key_type.clone(), - id: String::new(), - use_case: inner.use_case.clone(), - tracked: inner.tracked, - args: Vec::new(), - }; - let parts = Operation { - identity: identity.clone(), - policy: inner.policy.clone(), - budget: inner.budget, - codec: inner.codec.clone(), - comparator: inner.comparator.clone(), - should_recover: inner.should_recover.clone(), - preview: inner.preview.clone(), - } - .erase(); + let identity = inner.identity.clone(); + let provider_identity = identity.clone(); let provider_args = args.clone(); let provider_inner = inner.clone(); let identity_provider = Arc::new(move || -> Result { @@ -418,20 +398,15 @@ where Ok(Identity { id: spec.id, args, - ..identity.clone() + ..provider_identity.clone() }) }); let source_inner = inner.clone(); let load = erase_load(move |scope: Scope| (source_inner.source)(scope, args.clone())); let erased = ErasedOperation { - identity: parts.identity, + identity, identity_provider: Some(identity_provider), - policy: parts.policy, - budget: parts.budget, - codec: parts.codec, - compare: parts.compare, - should_recover: parts.should_recover, - preview: parts.preview, + metadata: inner.metadata.clone(), load, }; let value = inner.cache.execute(scope, erased).await?; diff --git a/rust/tests/default_logger.rs b/rust/tests/default_logger.rs new file mode 100644 index 00000000..6c7e8a22 --- /dev/null +++ b/rust/tests/default_logger.rs @@ -0,0 +1,42 @@ +//! Exercise the actual default log facade, including optional value previews. +use dialcache::observe::LogFacadeLogger; +use dialcache::{LogEvent, Logger, ShadowMismatchDetails}; +use std::sync::Mutex; + +struct Capture(Mutex>); +impl log::Log for Capture { + fn enabled(&self, _: &log::Metadata<'_>) -> bool { + true + } + fn log(&self, record: &log::Record<'_>) { + self.0.lock().unwrap().push(record.args().to_string()); + } + fn flush(&self) {} +} +static CAPTURE: Capture = Capture(Mutex::new(Vec::new())); + +#[test] +fn default_warning_includes_only_available_bounded_previews() { + log::set_logger(&CAPTURE).unwrap(); + log::set_max_level(log::LevelFilter::Warn); + for (cached, source) in [ + (Some("\"old\""), Some("\"new\"")), + (None, Some("null")), + (None, None), + ] { + LogFacadeLogger.log(&LogEvent::ShadowMismatch(ShadowMismatchDetails { + namespace: "app".into(), + use_case: "lookup".into(), + key_type: "thing".into(), + cache_key: "one".into(), + cached_value_json: cached.map(str::to_owned), + source_value_json: source.map(str::to_owned), + })); + } + let messages = CAPTURE.0.lock().unwrap(); + assert!(messages[0].contains("cachedValue=\"old\" sourceValue=\"new\"")); + assert!(!messages[1].contains("cachedValue=")); + assert!(messages[1].contains("sourceValue=null")); + assert!(!messages[2].contains("cachedValue=")); + assert!(!messages[2].contains("sourceValue=")); +} diff --git a/rust/tests/formal/causal.rs b/rust/tests/formal/causal.rs index 7308fe25..472c6031 100644 --- a/rust/tests/formal/causal.rs +++ b/rust/tests/formal/causal.rs @@ -69,6 +69,17 @@ impl Runtime for InvocationRuntime { .defer(Box::pin(InvocationFuture::new(current_invocation(), task))); } + fn spawn_blocking( + &self, + task: Box, + ) -> Result<(), dialcache::BoxError> { + let owner = current_invocation(); + self.0.spawn_blocking(Box::new(move || { + let _restore = RestoreInvocation(INVOCATION.replace(owner)); + task(); + })) + } + fn sleep(&self, duration: Duration) -> BoxFuture<'static, ()> { self.0.sleep(duration) } diff --git a/rust/tests/key_spec.rs b/rust/tests/key_spec.rs index 615433e3..5cdc8a6e 100644 --- a/rust/tests/key_spec.rs +++ b/rust/tests/key_spec.rs @@ -242,3 +242,54 @@ fn string_invalidation_preserves_explicit_entity_text() { assert_invalidation_matches_registered_id("001e+21", "001e%2B21"); assert_invalidation_matches_registered_id(String::from("-0"), "-0"); } + +#[test] +fn identity_invalidation_matches_explicit_and_inherited_namespaces_and_all_variants() { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let remote = Arc::new(RecordingRemote::default()); + let cache = DialCache::builder() + .namespace("tenant-a") + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote_arc(remote.clone()) + .build() + .unwrap(); + executor.block_on(async move { + let request = cache.enable_guard(); + for namespace in ["", "tenant-b"] { + for use_case in ["summary", "details"] { + let mut identity = Identity::new("thing", 42, use_case) + .namespace(namespace) + .tracked(true); + identity.args = vec![("variant".to_owned(), use_case.to_owned())]; + cache + .get_or_load( + request.scope(), + Operation::::new(identity.clone()) + .policy(Policy::default().remote_ttl_sec(60)), + |_| async { Ok(7) }, + ) + .await + .unwrap(); + cache.invalidate_identity(identity, 17).await.unwrap(); + } + } + }); + let reads = remote.reads.lock(); + let invalidations = remote.invalidations.lock(); + assert_eq!(reads.len(), 4); + assert_eq!(invalidations.len(), 4); + for (i, (read, invalidation)) in reads.iter().zip(invalidations.iter()).enumerate() { + assert_eq!( + read.watermark_key.as_ref(), + Some(&invalidation.watermark_key) + ); + let namespace = if i < 2 { "tenant-a" } else { "tenant-b" }; + assert_eq!( + invalidation.watermark_key, + format!("{{{namespace}:thing:42}}#watermark") + ); + assert_eq!(invalidation.future_buffer_ms, 17); + } + assert_ne!(reads[0].value_key, reads[1].value_key); +} diff --git a/rust/tests/operation_api.rs b/rust/tests/operation_api.rs index 17e619ed..1a2d65f2 100644 --- a/rust/tests/operation_api.rs +++ b/rust/tests/operation_api.rs @@ -45,3 +45,204 @@ fn cloned_operation_reuses_non_clone_cached_values() { assert!(Arc::ptr_eq(&first, &second)); assert_eq!(sources.load(Ordering::SeqCst), 1); } + +#[test] +fn incompatible_settled_memory_values_miss_and_are_replaced() { + use dialcache::observe::Layer; + use dialcache::{Event, MissReason, Observer}; + use parking_lot::Mutex; + #[derive(Default)] + struct Events(Mutex>); + impl Observer for Events { + fn observe(&self, event: &Event) { + self.0.lock().push(event.clone()); + } + } + for policy in [ + Policy::default().request_local(true), + Policy::default().local_ttl_sec(60), + ] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let events = Arc::new(Events::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .observer_arc(events.clone()) + .build() + .unwrap(); + executor.block_on(async move { + let request = cache.enable_guard(); + let id = Identity::new("thing", "one", "MixedTypes"); + cache + .get_or_load( + request.scope(), + Operation::::new(id.clone()).policy(policy.clone()), + |_| async { Ok("old".to_owned()) }, + ) + .await + .unwrap(); + let op = Operation::::new(id).policy(policy); + let value = cache + .get_or_load(request.scope(), op.clone(), |_| async { + Ok(serde_json::json!("new")) + }) + .await + .unwrap(); + assert_eq!(*value, serde_json::json!("new")); + let hit = cache + .get_or_load(request.scope(), op, |_| async { + panic!("typed entry should hit") + }) + .await + .unwrap(); + assert!(Arc::ptr_eq(&value, &hit)); + }); + assert!(events.0.lock().iter().any(|event| matches!(event, + Event::Miss { labels, reason: MissReason::Unclassified } + if matches!(labels.layer, Layer::Local | Layer::RequestLocal)))); + } +} + +#[test] +fn incompatible_memory_hit_can_decode_compatible_remote_json() { + use dialcache::{ + BoxError, Frame, InvalidateRequest, MissReason, ReadContext, ReadRequest, ReadResult, + Remote, WriteRequest, + }; + use futures::future::BoxFuture; + use parking_lot::Mutex; + #[derive(Default)] + struct MemoryRemote(Mutex>); + impl Remote for MemoryRemote { + fn read( + &self, + _: ReadRequest, + _: ReadContext, + ) -> BoxFuture<'_, Result> { + Box::pin(std::future::ready(Ok(self + .0 + .lock() + .clone() + .map(ReadResult::Hit) + .unwrap_or(ReadResult::miss(MissReason::ValueAbsent))))) + } + fn write(&self, value: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + *self.0.lock() = Some(value.frame); + Box::pin(std::future::ready(Ok(()))) + } + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(std::future::ready(Ok(()))) + } + } + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote(MemoryRemote::default()) + .build() + .unwrap(); + executor.block_on(async move { + let request = cache.enable_guard(); + let id = Identity::new("thing", "one", "CompatibleJson"); + let policy = Policy::enabled(60).request_local(true); + cache + .get_or_load( + request.scope(), + Operation::::new(id.clone()).policy(policy.clone()), + |_| async { Ok("stored".to_owned()) }, + ) + .await + .unwrap(); + let value = cache + .get_or_load( + request.scope(), + Operation::::new(id).policy(policy), + |_| async { panic!("compatible remote representation must be decoded") }, + ) + .await + .unwrap(); + assert_eq!(*value, serde_json::json!("stored")); + }); +} + +#[test] +fn registration_and_disabled_calls_do_not_evaluate_keys_or_runtime_policy() { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .policy_provider(|_| async { panic!("disabled policy resolution") }) + .build() + .unwrap(); + let lookup = cache + .use_case::<(), String>("thing", "LazyRegistration") + .key(|_| panic!("disabled key construction")) + .source(|_, _| async { Ok("source".to_owned()) }) + .register() + .unwrap(); + executor.block_on(async move { + assert_eq!(*lookup.get_uncached(()).await.unwrap(), "source"); + }); +} + +#[test] +fn incompatible_coalesced_follower_errors_without_retrying_the_source() { + use futures::channel::oneshot; + use parking_lot::Mutex; + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .build() + .unwrap(); + let guard = cache.enable_guard(); + let (send, receive) = oneshot::channel(); + let receive = Arc::new(Mutex::new(Some(receive))); + let first = cache.clone(); + let scope = guard.scope().clone(); + executor.spawn(async move { + first + .get_or_load( + &scope, + Operation::::new(Identity::new("thing", "one", "MixedFlight")) + .policy(Policy::default().local_ttl_sec(60)), + move |_| { + let receive = receive.lock().take().unwrap(); + async move { + receive.await.unwrap(); + Ok("first".to_owned()) + } + }, + ) + .await + .unwrap(); + }); + executor.drain(); + let result = Arc::new(Mutex::new(None)); + let sink = result.clone(); + let scope = guard.scope().clone(); + executor.spawn(async move { + *sink.lock() = Some( + cache + .get_or_load( + &scope, + Operation::::new(Identity::new( + "thing", + "one", + "MixedFlight", + )) + .policy(Policy::default().local_ttl_sec(60)), + |_| async { panic!("follower retried source") }, + ) + .await, + ); + }); + executor.drain(); + assert!(result.lock().is_none()); + send.send(()).unwrap(); + executor.drain(); + assert!(matches!( + result.lock().take().unwrap(), + Err(dialcache::Error::Config(_)) + )); +} From 974f3f592326d984de132ad09fdef4a6c2bffc13 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Mon, 21 Sep 2026 18:37:18 -0700 Subject: [PATCH 27/31] perf(rust): bound mismatch previews and simplify timeouts --- rust/README.md | 9 +- rust/src/cpu_tests.rs | 204 ++++++++++++++++++++++++++++++ rust/src/execution.rs | 10 +- rust/src/operation.rs | 4 +- rust/src/preview.rs | 102 +++++++++++++++ rust/src/shadow.rs | 72 ++++++----- rust/src/use_case.rs | 4 +- rust/tests/callback_boundaries.rs | 74 +++++++++++ 8 files changed, 437 insertions(+), 42 deletions(-) diff --git a/rust/README.md b/rust/README.md index 62cf7405..e2599aa6 100644 --- a/rust/README.md +++ b/rust/README.md @@ -206,8 +206,13 @@ validation exists only to be observed, so a job is admitted only when the observer opts in through `observes_shadow_outcomes`; the bundled exporters do. `Logger` receives structured `LogEvent`s and defaults to the `log` facade. Mismatch logging is opt-in, confirmed, bounded, and previews values through -the operation's `preview` (JSON for serde values). Observer and logger failures -never change a cache, source or maintenance result. +the operation's `preview` (JSON for serde values). Default JSON previews retain +only an 8 KiB prefix while checking the entire serialization for errors. Preview +callbacks run through the bounded CPU executor after confirmation; they may run +on a worker thread. Queue rejection omits value previews but still logs the +confirmed mismatch. Diagnostic work holds shadow capacity until it finishes, +including after runtime shutdown. Observer and logger failures never change a +cache, source or maintenance result. `MetricKind` maps every event to the metric names, labels and values shared with the TypeScript and Go exporters. `PrometheusObserver` (feature diff --git a/rust/src/cpu_tests.rs b/rust/src/cpu_tests.rs index 680db718..ed24409a 100644 --- a/rust/src/cpu_tests.rs +++ b/rust/src/cpu_tests.rs @@ -334,3 +334,207 @@ async fn large_codec_phases_allow_an_independent_short_timer_to_progress() { assert_eq!(decodes.load(Ordering::SeqCst), usize::from(decode)); } } + +#[derive(Default)] +struct Logs(Mutex>); +impl Logger for Logs { + fn log(&self, event: &LogEvent) { + if let LogEvent::ShadowMismatch(details) = event { + self.0.lock().push(details.clone()); + } + } +} +fn preview_setup() -> ( + TestExecutor, + DialCache, + Arc, + Arc, + Arc, +) { + let executor = TestExecutor::new(WALL_EPOCH_MS); + let runtime = Arc::new(CpuRuntime { + step: executor.runtime.clone(), + jobs: Mutex::new(VecDeque::new()), + reject: AtomicBool::new(false), + }); + let events = Arc::new(Events::default()); + let logs = Arc::new(Logs::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(runtime.clone()) + .observer_arc(events.clone()) + .logger_arc(logs.clone()) + .remote(SnapshotRemote { + result: ReadResult::Hit(Frame { + created_at_ms: WALL_EPOCH_MS as u64, + payload: Payload::text(serde_json::to_string(&"é".repeat(10_000)).unwrap()), + }), + writes: AtomicUsize::new(0), + }) + .build() + .unwrap(); + (executor, cache, runtime, events, logs) +} +fn preview_call( + executor: &mut TestExecutor, + cache: &DialCache, + registered: bool, + preview: Option>, +) { + let cache = cache.clone(); + executor.block_on(async move { + let guard = cache.enable_guard(); + let policy = Policy::default().remote_ttl_sec(60).shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: Some(true), + }); + let result = if registered { + let use_case = cache + .use_case::<(), String>("thing", "Preview") + .policy(policy) + .key(|_| KeySpec::new("one")) + .source(|_, ()| async { Ok("source".to_owned()) }) + .register() + .unwrap(); + use_case.get(guard.scope(), ()).await + } else { + let mut operation = + Operation::::new(Identity::new("thing", "one", "Preview")).policy(policy); + if let Some(preview) = preview { + operation.preview = Some(preview); + } + cache + .get_or_load(guard.scope(), operation, |_| async { + Ok("source".to_owned()) + }) + .await + }; + assert_eq!(*result.unwrap(), "é".repeat(10_000)); + }); +} + +#[test] +fn both_default_apis_prepare_bounded_previews_without_holding_the_caller() { + for registered in [false, true] { + let (mut executor, cache, runtime, events, logs) = preview_setup(); + preview_call(&mut executor, &cache, registered, None); + assert_eq!(*events.0.lock(), vec![ShadowOutcome::Mismatch]); + assert!(logs.0.lock().is_empty()); + assert_eq!(runtime.jobs.lock().len(), 1); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + // An inline call uses the same identity as the registered handle. + preview_call(&mut executor, &cache, false, None); + assert_eq!( + *events.0.lock(), + vec![ShadowOutcome::Mismatch, ShadowOutcome::Dropped] + ); + executor.advance(10_000, true); + runtime.take()(); + executor.drain(); + assert!(cache.core.state.lock().shadows.is_empty()); + let messages = logs.0.lock(); + assert_eq!(messages.len(), 1); + assert_eq!( + messages[0].cached_value_json, + crate::preview::json_preview(&"é".repeat(10_000)) + ); + assert_eq!(messages[0].source_value_json.as_deref(), Some("\"source\"")); + assert_eq!( + *events.0.lock(), + vec![ShadowOutcome::Mismatch, ShadowOutcome::Dropped] + ); + } +} + +#[test] +fn rejected_preview_cpu_work_still_logs_the_confirmed_mismatch() { + let (mut executor, cache, runtime, events, logs) = preview_setup(); + runtime.reject.store(true, Ordering::SeqCst); + preview_call(&mut executor, &cache, false, None); + assert_eq!(*events.0.lock(), vec![ShadowOutcome::Mismatch]); + assert!(runtime.jobs.lock().is_empty()); + assert!(cache.core.state.lock().shadows.is_empty()); + let messages = logs.0.lock(); + assert_eq!(messages.len(), 1); + assert!(messages[0].cached_value_json.is_none()); + assert!(messages[0].source_value_json.is_none()); +} + +#[test] +fn custom_preview_failures_are_isolated_per_value_and_success_is_clamped() { + for panic in [false, true] { + let (mut executor, cache, runtime, _, logs) = preview_setup(); + preview_call( + &mut executor, + &cache, + false, + Some(Arc::new(move |value| { + if value == "source" { + Some("🙂".repeat(10_000)) + } else if panic { + panic!("bad cached preview"); + } else { + None + } + })), + ); + runtime.take()(); + executor.drain(); + let messages = logs.0.lock(); + assert_eq!(messages.len(), 1); + assert!(messages[0].cached_value_json.is_none()); + assert_eq!( + messages[0].source_value_json, + Some(crate::preview::preview_value(&"🙂".repeat(10_000))) + ); + assert!(cache.core.state.lock().shadows.is_empty()); + } +} + +#[test] +fn queued_preview_keeps_capacity_after_executor_shutdown_until_run_or_discarded() { + for discard in [false, true] { + let (mut executor, cache, runtime, _, _) = preview_setup(); + preview_call(&mut executor, &cache, false, None); + drop(executor); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + let job = runtime.take(); + if discard { + drop(job); + } else { + job(); + } + assert!(cache.core.state.lock().shadows.is_empty()); + } +} + +#[test] +fn running_preview_keeps_capacity_after_executor_shutdown() { + let (mut executor, cache, runtime, _, _) = preview_setup(); + let (started, running) = std::sync::mpsc::channel(); + let (release, gate) = std::sync::mpsc::channel(); + let gate = Mutex::new(gate); + preview_call( + &mut executor, + &cache, + false, + Some(Arc::new(move |value| { + if value != "source" { + started.send(std::thread::current().id()).unwrap(); + gate.lock().recv_timeout(Duration::from_secs(5)).unwrap(); + } + Some(value.clone()) + })), + ); + let job = runtime.take(); + let worker = std::thread::spawn(job); + assert_ne!( + running.recv_timeout(Duration::from_secs(5)).unwrap(), + std::thread::current().id() + ); + drop(executor); + assert_eq!(cache.core.state.lock().shadows.len(), 1); + release.send(()).unwrap(); + worker.join().unwrap(); + assert!(cache.core.state.lock().shadows.is_empty()); +} diff --git a/rust/src/execution.rs b/rust/src/execution.rs index c86ce9a9..3e86d5ef 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -3,7 +3,6 @@ //! implementations; the Quint models define the behavior. use std::panic::{catch_unwind, AssertUnwindSafe}; -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -108,7 +107,6 @@ pub(crate) struct Execution { pub(crate) keys: Keys, pub(crate) policy: ResolvedPolicy, pub(crate) labels: OutcomeLabels, - pub(crate) timed_out: Arc, } /// What the serving remote read produced. @@ -245,7 +243,6 @@ pub(crate) async fn run(core: Arc, scope: Scope, op: Arc) keys, policy, labels, - timed_out: Arc::new(AtomicBool::new(false)), }); if !execution.policy.request_local { return execution.shared(Layer::Local).await; @@ -314,7 +311,7 @@ async fn uncached_source( scope: Scope, labels: &OutcomeLabels, ) -> ValueResult { - source_with_budget(core, op, scope, labels, Layer::Noop, None).await + source_with_budget(core, op, scope, labels, Layer::Noop).await } /// Run the source under the operation budget, reporting the fallback trail. @@ -324,7 +321,6 @@ async fn source_with_budget( scope: Scope, labels: &OutcomeLabels, layer: Layer, - timed_out: Option<&AtomicBool>, ) -> ValueResult { let clock = core.clock.clone(); let start = clock.elapsed(); @@ -345,9 +341,6 @@ async fn source_with_budget( start, budget, || { - if let Some(flag) = timed_out { - flag.store(true, Ordering::SeqCst); - } Err(Error::FallbackTimeout(Arc::new(FallbackTimeout { use_case, timeout_ms: budget.unwrap_or(0), @@ -406,7 +399,6 @@ impl Execution { self.scope.clone(), &self.labels, layer, - Some(&self.timed_out), ) .await } diff --git a/rust/src/operation.rs b/rust/src/operation.rs index 615dc025..def6b255 100644 --- a/rust/src/operation.rs +++ b/rust/src/operation.rs @@ -44,6 +44,8 @@ pub type RecoveryPredicate = Arc bool + Send + Sync>; /// Application equality used by shadow validation. pub type Comparator = Arc bool + Send + Sync>; /// Produces a bounded textual preview of a value for mismatch warnings. +/// Runs through [`Runtime::spawn_blocking`](crate::Runtime::spawn_blocking) +/// after mismatch confirmation; callbacks may run on a CPU worker thread. pub type Preview = Arc Option + Send + Sync>; /// One inline cached call: its identity, static policy, budget and codecs. @@ -104,7 +106,7 @@ impl Operat codec: Arc::new(JsonCodec), comparator: Arc::new(|a: &T, b: &T| a == b), should_recover: None, - preview: Some(Arc::new(|value: &T| serde_json::to_string(value).ok())), + preview: Some(Arc::new(crate::preview::json_preview::)), } } } diff --git a/rust/src/preview.rs b/rust/src/preview.rs index e7ae5cb9..93efe84c 100644 --- a/rust/src/preview.rs +++ b/rust/src/preview.rs @@ -1,5 +1,9 @@ //! Bounded previews for mismatch warnings. +use std::io::{self, Write}; + +use serde::Serialize; + /// Maximum UTF-8 bytes of a logged cache key. pub const SHADOW_LOG_KEY_MAX_BYTES: usize = 2 * 1024; /// Maximum UTF-8 bytes of a logged value preview. @@ -7,6 +11,60 @@ pub const SHADOW_LOG_VALUE_MAX_BYTES: usize = 8 * 1024; /// Marker appended to a clamped preview. pub const SHADOW_LOG_TRUNCATION_MARKER: &str = "...[truncated]"; +/// Retain only the logged prefix, but consume the complete serialization so +/// errors beyond the cap still suppress the preview. +struct JsonPrefix { + bytes: Vec, + truncated: bool, +} + +impl JsonPrefix { + fn new() -> Self { + Self { + bytes: Vec::with_capacity(SHADOW_LOG_VALUE_MAX_BYTES), + truncated: false, + } + } + + fn finish(mut self) -> String { + if self.truncated { + self.bytes + .truncate(SHADOW_LOG_VALUE_MAX_BYTES - SHADOW_LOG_TRUNCATION_MARKER.len()); + } + // The prefix can end inside a multibyte character. JSON serialization + // produces valid UTF-8, so only that trailing character can be partial. + if let Err(error) = std::str::from_utf8(&self.bytes) { + self.bytes.truncate(error.valid_up_to()); + } + let mut text = String::from_utf8(self.bytes).expect("valid JSON prefix"); + if self.truncated { + text.push_str(SHADOW_LOG_TRUNCATION_MARKER); + } + text + } +} + +impl Write for JsonPrefix { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let retained = bytes + .len() + .min(SHADOW_LOG_VALUE_MAX_BYTES - self.bytes.len()); + self.bytes.extend_from_slice(&bytes[..retained]); + self.truncated |= retained < bytes.len(); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +pub(crate) fn json_preview(value: &T) -> Option { + let mut prefix = JsonPrefix::new(); + serde_json::to_writer(&mut prefix, value).ok()?; + Some(prefix.finish()) +} + /// Clamp text to `max_bytes` of UTF-8, ending a clamped result with the /// truncation marker on a character boundary. pub fn clamp_utf8(value: &str, max_bytes: usize) -> String { @@ -36,6 +94,7 @@ pub fn preview_value(json: &str) -> String { #[cfg(test)] mod tests { use super::*; + use serde::ser::{Error, SerializeSeq}; #[test] fn clamps_on_character_boundaries() { @@ -45,4 +104,47 @@ mod tests { assert!(clamped.ends_with(SHADOW_LOG_TRUNCATION_MARKER)); assert_eq!(clamp_utf8("short", 100), "short"); } + + #[test] + fn streamed_json_preserves_the_existing_preview_prefix() { + for value in [ + "short".to_owned(), + "x".repeat(SHADOW_LOG_VALUE_MAX_BYTES - 2), + "x".repeat(SHADOW_LOG_VALUE_MAX_BYTES - 1), + "é🙂".repeat(SHADOW_LOG_VALUE_MAX_BYTES), + "\n\t\"\\".repeat(SHADOW_LOG_VALUE_MAX_BYTES), + ] { + assert_eq!( + json_preview(&value).unwrap(), + preview_value(&serde_json::to_string(&value).unwrap()) + ); + } + } + + struct LargeSequence { + fail_at_end: bool, + } + + impl Serialize for LargeSequence { + fn serialize(&self, serializer: S) -> Result { + let mut sequence = serializer.serialize_seq(Some(10_000))?; + for _ in 0..10_000 { + sequence.serialize_element(&"large element".repeat(100))?; + } + if self.fail_at_end { + return Err(S::Error::custom("failure after the logged prefix")); + } + sequence.end() + } + } + + #[test] + fn large_json_construction_keeps_a_capped_buffer_and_detects_late_errors() { + let mut prefix = JsonPrefix::new(); + serde_json::to_writer(&mut prefix, &LargeSequence { fail_at_end: false }).unwrap(); + assert_eq!(prefix.bytes.len(), SHADOW_LOG_VALUE_MAX_BYTES); + assert_eq!(prefix.bytes.capacity(), SHADOW_LOG_VALUE_MAX_BYTES); + assert!(prefix.finish().ends_with(SHADOW_LOG_TRUNCATION_MARKER)); + assert!(json_preview(&LargeSequence { fail_at_end: true }).is_none()); + } } diff --git a/rust/src/shadow.rs b/rust/src/shadow.rs index ea4d0a64..faa92ef8 100644 --- a/rust/src/shadow.rs +++ b/rust/src/shadow.rs @@ -153,7 +153,7 @@ impl Execution { || state.shadows.len() >= self.core.shadow_max_in_flight { drop(state); - self.shadow_event(Verdict::of(ShadowOutcome::Dropped)); + self.shadow_event(&Verdict::of(ShadowOutcome::Dropped)); return; } let flight = ShadowFlight::new(); @@ -176,7 +176,7 @@ impl Execution { })); } - fn shadow_event(&self, verdict: Verdict) { + fn shadow_event(&self, verdict: &Verdict) { self.emit(Event::ShadowValidation { labels: self.labels.clone(), outcome: verdict.outcome, @@ -188,28 +188,42 @@ impl Execution { seconds: age.max(0) as f64 / 1000.0, }); } - if verdict.outcome == ShadowOutcome::Mismatch && self.policy.shadow.log_mismatches { - let preview = |value: &StoredValue| -> Option { - let preview = self.op.metadata.preview.as_ref()?; - catch_unwind(AssertUnwindSafe(|| preview(value))) - .ok() - .flatten() - .map(|json| preview_value(&json)) - }; - let (cached, source) = match &verdict.compared { - Some((cached, source)) => (preview(cached), preview(source)), - None => (None, None), - }; - self.core - .log(LogEvent::ShadowMismatch(ShadowMismatchDetails { - namespace: self.labels.namespace.clone(), - use_case: self.labels.use_case.clone(), - key_type: self.labels.key_type.clone(), - cache_key: preview_key(&self.keys.logical), - cached_value_json: cached, - source_value_json: source, - })); + } + + async fn log_shadow_mismatch(&self, verdict: Verdict, slot: Arc) { + if verdict.outcome != ShadowOutcome::Mismatch || !self.policy.shadow.log_mismatches { + return; } + let (cached, source) = match (self.op.metadata.preview.clone(), verdict.compared) { + (Some(preview), Some((cached, source))) => { + let cpu_slot = slot.clone(); + crate::blocking::run(self.core.runtime.as_ref(), move || { + // Raw diagnostic work keeps capacity even if the awaiting + // async task is dropped during runtime shutdown. + let _slot = cpu_slot; + let render = |value: &StoredValue| { + catch_unwind(AssertUnwindSafe(|| preview(value))) + .ok() + .flatten() + .map(|json| preview_value(&json)) + }; + Ok((render(&cached), render(&source))) + }) + .await + .unwrap_or_default() + } + _ => (None, None), + }; + self.core + .log(LogEvent::ShadowMismatch(ShadowMismatchDetails { + namespace: self.labels.namespace.clone(), + use_case: self.labels.use_case.clone(), + key_type: self.labels.key_type.clone(), + cache_key: preview_key(&self.keys.logical), + cached_value_json: cached, + source_value_json: source, + })); + drop(slot); } async fn run_shadow( @@ -239,6 +253,7 @@ impl Execution { started, budget: Duration::from_millis(budget_ms), }; + let log_slot = slot.clone(); let reads: Arc>>> = Arc::new(Mutex::new(Vec::new())); let validation: Settled = start_pending( self.core.runtime.as_ref(), @@ -287,7 +302,8 @@ impl Execution { }, ) .await; - self.shadow_event(verdict); + self.shadow_event(&verdict); + self.log_shadow_mismatch(verdict, log_slot).await; } async fn shadow_read( @@ -390,12 +406,10 @@ impl Execution { }; let value = match value { Ok(value) => value, - Err(_) => { - if source.is_some() && self.timed_out.load(Ordering::SeqCst) { - return Verdict::of(ShadowOutcome::Timeout); - } - return Verdict::of(ShadowOutcome::SourceError); + Err(Error::FallbackTimeout(_)) if source.is_some() => { + return Verdict::of(ShadowOutcome::Timeout); } + Err(_) => return Verdict::of(ShadowOutcome::SourceError), }; if expired() { return Verdict::of(ShadowOutcome::Timeout); diff --git a/rust/src/use_case.rs b/rust/src/use_case.rs index b41613e0..a814e0fa 100644 --- a/rust/src/use_case.rs +++ b/rust/src/use_case.rs @@ -227,6 +227,8 @@ where } /// Bounded textual preview of a value for mismatch warnings. + /// Runs through [`Runtime::spawn_blocking`](crate::Runtime::spawn_blocking) + /// after confirmation; callbacks may run on a CPU worker thread. pub fn preview( mut self, preview: impl Fn(&T) -> Option + Send + Sync + 'static, @@ -331,7 +333,7 @@ where let preview = self .preview .clone() - .or_else(|| Some(Arc::new(|value: &T| serde_json::to_string(value).ok()))); + .or_else(|| Some(Arc::new(crate::preview::json_preview::))); self.finish(codec, comparator, preview) } } diff --git a/rust/tests/callback_boundaries.rs b/rust/tests/callback_boundaries.rs index c1370327..b7844cf3 100644 --- a/rust/tests/callback_boundaries.rs +++ b/rust/tests/callback_boundaries.rs @@ -238,3 +238,77 @@ fn synchronous_shadow_source_panic_is_a_source_error_and_releases_capacity() { vec![ShadowOutcome::SourceError, ShadowOutcome::SourceError] ); } + +#[test] +fn dark_shadow_distinguishes_its_deadline_from_application_timeout_errors() { + for error_kind in ["own deadline", "nested deadline", "source error"] { + let (mut executor, cache, _, events) = setup(hit(0)); + let result = Arc::new(Mutex::new(None)); + let sink = result.clone(); + executor.spawn(async move { + let request = cache.enable_guard(); + let operation = Operation::::new(Identity::new("thing", "one", "DarkDeadline")) + .policy( + Policy::default() + .remote_ttl_sec(60) + .remote_ramp(0.0) + .shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: None, + }), + ) + .budget(dialcache::SourceBudget::Millis(5)); + *sink.lock() = Some( + cache + .get_or_load(request.scope(), operation, move |_| async move { + match error_kind { + "own deadline" => std::future::pending::>().await, + "nested deadline" => Err(Box::new(dialcache::Error::FallbackTimeout( + Arc::new(dialcache::FallbackTimeout { + use_case: "nested".to_owned(), + timeout_ms: 1, + }), + )) as BoxError), + _ => Err("source failed".into()), + } + }) + .await, + ); + }); + executor.drain(); + if error_kind == "own deadline" { + assert!(result.lock().is_none()); + executor.advance(5, true); + assert!(matches!( + result.lock().as_ref().unwrap(), + Err(dialcache::Error::FallbackTimeout(_)) + )); + } else { + assert!(matches!( + result.lock().as_ref().unwrap(), + Err(dialcache::Error::Source(_)) + )); + } + let outcomes: Vec<_> = events + .0 + .lock() + .iter() + .filter_map(|event| { + if let Event::ShadowValidation { outcome, .. } = event { + Some(*outcome) + } else { + None + } + }) + .collect(); + assert_eq!( + outcomes, + vec![if error_kind == "own deadline" { + ShadowOutcome::Timeout + } else { + ShadowOutcome::SourceError + }], + "{error_kind}" + ); + } +} From b8117ca02f84d55a78fcb71f511a7a729e3fb674 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Mon, 21 Sep 2026 19:05:34 -0700 Subject: [PATCH 28/31] perf(rust): resolve policy in the existing detached invocation --- rust/src/execution.rs | 25 ++--- rust/tests/callback_boundaries.rs | 161 +++++++++++++++++++++++++++++- 2 files changed, 166 insertions(+), 20 deletions(-) diff --git a/rust/src/execution.rs b/rust/src/execution.rs index 3e86d5ef..280af67f 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -180,25 +180,12 @@ pub(crate) async fn run(core: Arc, scope: Scope, op: Arc) } }; let overlay: Result, BoxError> = match &core.provider { - Some(provider) => { - let pending = start_pending( - core.runtime.as_ref(), - { - let provider = provider.clone(); - let identity = identity.clone(); - async move { - provider(identity) - .await - .map_err(|e| Arc::from(e) as SharedError) - } - }, - |message| Err(Arc::new(PanicError(message)) as SharedError), - ); - pending - .wait() - .await - .map_err(|e| Box::new(SharedErrorWrapper(e)) as BoxError) - } + Some(provider) => AssertUnwindSafe(async { provider(identity.clone()).await }) + .catch_unwind() + .await + .unwrap_or_else( + |payload| Err(Box::new(PanicError(panic_message(payload))) as BoxError), + ), None => Ok(None), }; let resolved = overlay.and_then(|overlay| { diff --git a/rust/tests/callback_boundaries.rs b/rust/tests/callback_boundaries.rs index b7844cf3..698250e3 100644 --- a/rust/tests/callback_boundaries.rs +++ b/rust/tests/callback_boundaries.rs @@ -5,7 +5,7 @@ use std::future::{ready, Ready}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::Arc; -use dialcache::observe::{ErrorKind, RecoveryOutcome, ShadowOutcome}; +use dialcache::observe::{ErrorKind, Layer, RecoveryOutcome, ShadowOutcome}; use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; use dialcache::{ BoxError, DialCache, Event, Frame, FromSync, Identity, InvalidateRequest, JsonCodec, @@ -312,3 +312,162 @@ fn dark_shadow_distinguishes_its_deadline_from_application_timeout_errors() { ); } } + +#[test] +fn provider_construction_poll_and_returned_failures_preserve_source_results() { + for failure in ["construct", "poll", "error"] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let events = Arc::new(Events::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .observer_arc(events.clone()) + .policy_provider( + move |_| -> BoxFuture<'static, Result, BoxError>> { + if failure == "construct" { + panic!("provider future construction"); + } + Box::pin(async move { + if failure == "poll" { + panic!("provider future poll"); + } + Err("provider failed".into()) + }) + }, + ) + .build() + .unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let request = cache.enable_guard(); + for expected in 1..=2 { + let (cache, scope, calls) = (cache.clone(), request.scope().clone(), calls.clone()); + let value = executor.block_on(async move { + cache + .get_or_load( + &scope, + Operation::::new(Identity::new("thing", "one", "ProviderFailure")) + .policy(Policy::default().local_ttl_sec(60)), + move |_| { + let value = calls.fetch_add(1, Ordering::SeqCst) + 1; + async move { Ok(value) } + }, + ) + .await + .unwrap() + }); + assert_eq!( + *value, expected, + "provider failure must bypass cache publication" + ); + } + let policy_errors = events.0.lock().iter().filter(|event| matches!(event, + Event::Error { labels, error: ErrorKind::ConfigResolution, .. } if labels.layer == Layer::Noop + )).count(); + assert_eq!(policy_errors, 2, "{failure}"); + } +} + +#[test] +fn held_policy_survives_caller_cancellation_and_rechecks_scope_closure() { + for close_scope in [false, true] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let (release, held) = futures::channel::oneshot::channel::<()>(); + let held = Arc::new(Mutex::new(Some(held))); + let provider_calls = Arc::new(AtomicUsize::new(0)); + let events = Arc::new(Events::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .observer_arc(events.clone()) + .policy_provider({ + let provider_calls = provider_calls.clone(); + move |_| { + provider_calls.fetch_add(1, Ordering::SeqCst); + let held = held.lock().take(); + async move { + if let Some(held) = held { + held.await.unwrap(); + } + Ok(None) + } + } + }) + .build() + .unwrap(); + let mut request = Some(cache.enable_guard()); + let scope = request.as_ref().unwrap().scope().clone(); + let operation = Operation::::new(Identity::new("thing", "one", "HeldProvider")) + .policy(Policy::default().local_ttl_sec(60)); + let calls = Arc::new(AtomicUsize::new(0)); + let source_enabled = Arc::new(Mutex::new(Vec::new())); + let caller_finished = Arc::new(AtomicBool::new(false)); + let (cancel, registration) = futures::future::AbortHandle::new_pair(); + executor.spawn({ + let (cache, operation, calls) = (cache.clone(), operation.clone(), calls.clone()); + let (source_enabled, caller_finished) = + (source_enabled.clone(), caller_finished.clone()); + async move { + let pending = cache.get_or_load(&scope, operation, move |scope| { + calls.fetch_add(1, Ordering::SeqCst); + source_enabled.lock().push(scope.is_enabled()); + async { Ok(7) } + }); + let result = futures::future::Abortable::new(pending, registration).await; + assert_eq!(result.is_err(), !close_scope); + if let Ok(value) = result { + assert_eq!(*value.unwrap(), 7); + } + caller_finished.store(true, Ordering::SeqCst); + } + }); + executor.drain(); + assert_eq!(provider_calls.load(Ordering::SeqCst), 1); + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert!(!caller_finished.load(Ordering::SeqCst)); + if close_scope { + drop(request.take()); + } else { + cancel.abort(); + } + executor.drain(); + assert_eq!(caller_finished.load(Ordering::SeqCst), !close_scope); + release.send(()).unwrap(); + executor.drain(); + assert!(caller_finished.load(Ordering::SeqCst)); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(*source_enabled.lock(), vec![!close_scope]); + let source_layers: Vec<_> = events + .0 + .lock() + .iter() + .filter_map(|event| match event { + Event::Fallback { labels, .. } => Some(labels.layer), + _ => None, + }) + .collect(); + assert_eq!( + source_layers, + vec![if close_scope { + Layer::Noop + } else { + Layer::Local + }] + ); + let calls_for_next = calls.clone(); + let value = executor.block_on(async move { + let request = cache.enable_guard(); + cache + .get_or_load(request.scope(), operation, move |_| { + calls_for_next.fetch_add(1, Ordering::SeqCst); + async { Ok(8) } + }) + .await + .unwrap() + }); + assert_eq!(*value, if close_scope { 8 } else { 7 }); + assert_eq!( + calls.load(Ordering::SeqCst), + if close_scope { 2 } else { 1 } + ); + } +} From 078cb35b0fa73b68036bbe88bb7b45ed1c213985 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Mon, 21 Sep 2026 19:51:48 -0700 Subject: [PATCH 29/31] feat(rust): expose shared ownership to custom encoders --- rust/README.md | 9 + rust/src/codec.rs | 16 ++ rust/src/execution.rs | 2 +- rust/src/operation.rs | 10 +- rust/tests/owned_codec.rs | 338 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 369 insertions(+), 6 deletions(-) create mode 100644 rust/tests/owned_codec.rs diff --git a/rust/README.md b/rust/README.md index e2599aa6..d1db4e22 100644 --- a/rust/README.md +++ b/rust/README.md @@ -201,6 +201,15 @@ until its admitted CPU work finishes or is discarded, including after a deadline or async runtime shutdown. Codecs supplied by the application still choose their own scheduling; `FromSync` does not offload application serialization. +For remote writes, the engine calls `Codec::encode_owned` with the existing +`Arc`. Its default implementation delegates to `encode(&T)`, so existing +codecs work unchanged. Override `encode_owned` to send a non-`Clone` value to a +background CPU job without copying it or changing the cached value type. +`decode` already receives an owned `Payload`. Application codecs own admission +and the lifetime of jobs they start; those jobs do not inherit the library's +shadow-capacity token. Default JSON encoding/decoding and `FromSync` remain +synchronous, so large values can occupy an async worker. + `Observer` receives every public diagnostic as a typed `Event`. Shadow validation exists only to be observed, so a job is admitted only when the observer opts in through `observes_shadow_outcomes`; the bundled exporters do. diff --git a/rust/src/codec.rs b/rust/src/codec.rs index 433314da..edcd1fd4 100644 --- a/rust/src/codec.rs +++ b/rust/src/codec.rs @@ -1,6 +1,7 @@ //! Serialized payloads and value codecs. use std::borrow::Cow; +use std::sync::Arc; use futures::future::BoxFuture; use serde::de::DeserializeOwned; @@ -64,6 +65,21 @@ pub trait Codec: Send + Sync + 'static { /// Serialize `value`. A failure counts as a `serialization_dump` error /// and skips the remote write. fn encode<'a>(&'a self, value: &'a T) -> BoxFuture<'a, Result>; + + /// Serialize a shared value owned by the cache. + /// + /// The engine calls this method for remote writes. By default it delegates + /// to [`encode`](Self::encode) without copying the value. Override it to + /// move the shared handle into a background job without requiring `T: Clone`. + /// The codec owns scheduling, admission and the lifetime of any jobs it + /// starts; those jobs do not inherit the cache's shadow-capacity token. + fn encode_owned(&self, value: Arc) -> BoxFuture<'_, Result> + where + T: Send + Sync + 'static, + { + Box::pin(async move { self.encode(value.as_ref()).await }) + } + /// Deserialize `payload` into an independent value. A failure counts as /// a `serialization_load` error and is treated as a miss. fn decode(&self, payload: Payload) -> BoxFuture<'_, Result>; diff --git a/rust/src/execution.rs b/rust/src/execution.rs index 280af67f..c56336f5 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -899,7 +899,7 @@ impl Execution { } let start = self.elapsed(); let codec = self.op.metadata.codec.clone(); - let encoded = match AssertUnwindSafe(async { codec.encode(value).await }) + let encoded = match AssertUnwindSafe(async { codec.encode(value.clone()).await }) .catch_unwind() .await { diff --git a/rust/src/operation.rs b/rust/src/operation.rs index def6b255..4e09a280 100644 --- a/rust/src/operation.rs +++ b/rust/src/operation.rs @@ -216,7 +216,7 @@ pub(crate) struct ErasedOperation { /// A codec over erased values. pub(crate) trait ErasedCodec: Send + Sync { - fn encode<'a>(&'a self, value: &'a StoredValue) -> BoxFuture<'a, Result>; + fn encode(&self, value: StoredValue) -> BoxFuture<'_, Result>; fn decode(&self, payload: Payload) -> BoxFuture<'_, Result>; } @@ -225,10 +225,10 @@ struct TypedCodec { } impl ErasedCodec for TypedCodec { - fn encode<'a>(&'a self, value: &'a StoredValue) -> BoxFuture<'a, Result> { - match value.downcast_ref::() { - Some(value) => self.codec.encode(value), - None => Box::pin(std::future::ready(Err( + fn encode(&self, value: StoredValue) -> BoxFuture<'_, Result> { + match Arc::downcast::(value) { + Ok(value) => self.codec.encode_owned(value), + Err(_) => Box::pin(std::future::ready(Err( "codec received a value of another type".into(), ))), } diff --git a/rust/tests/owned_codec.rs b/rust/tests/owned_codec.rs new file mode 100644 index 00000000..001410c9 --- /dev/null +++ b/rust/tests/owned_codec.rs @@ -0,0 +1,338 @@ +//! Shared ownership at the public codec boundary, without Clone or Serde bounds. + +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use dialcache::observe::{ErrorKind, SerializationOperation}; +use dialcache::testing::{StepRuntime, TestExecutor, WALL_EPOCH_MS}; +use dialcache::{ + BoxError, Codec, DialCache, Event, Frame, Identity, InvalidateRequest, KeySpec, MissReason, + Observer, Operation, Payload, Policy, ReadContext, ReadRequest, ReadResult, Remote, Runtime, + Scope, UseCase, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; + +#[derive(Debug, PartialEq)] +struct Value(u64); + +fn payload(value: &Value) -> Payload { + Payload::text(format!("custom:{}", value.0)) +} + +#[derive(Default)] +struct RecordingRemote(Mutex>); +impl Remote for RecordingRemote { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(async { Ok(ReadResult::miss(MissReason::ValueAbsent)) }) + } + fn write(&self, request: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + self.0.lock().push(request.frame); + Box::pin(async { Ok(()) }) + } + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(async { Ok(()) }) + } +} + +enum Lookup { + Inline(DialCache, Box>, Arc), + Registered(UseCase<(), Value>), +} +impl Lookup { + fn new( + cache: &DialCache, + codec: Arc>, + registered: bool, + calls: Arc, + ) -> Self { + let policy = Policy::enabled(60).request_local(false); + if registered { + Self::Registered( + cache + .use_case("thing", "OwnedCodec") + .policy(policy) + .key(|_: &()| KeySpec::new("one")) + .codec(codec) + .comparator(|a, b| a == b) + .source(move |_, _| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(Value(7)) } + }) + .register_custom() + .unwrap(), + ) + } else { + Self::Inline( + cache.clone(), + Box::new( + Operation::with_codec( + Identity::new("thing", "one", "OwnedCodec"), + codec, + |a, b| a == b, + ) + .policy(policy), + ), + calls, + ) + } + } + + async fn get(&self, scope: &Scope) -> Arc { + match self { + Self::Inline(cache, operation, calls) => { + let calls = calls.clone(); + cache + .get_or_load(scope, operation.as_ref().clone(), move |_| { + calls.fetch_add(1, Ordering::SeqCst); + async { Ok(Value(7)) } + }) + .await + } + Self::Registered(lookup) => lookup.get(scope, ()).await, + } + .expect("cache plumbing preserves the source result") + } +} + +struct BorrowedCodec(Arc); +impl Codec for BorrowedCodec { + fn encode<'a>(&'a self, value: &'a Value) -> BoxFuture<'a, Result> { + self.0.fetch_add(1, Ordering::SeqCst); + Box::pin(async move { Ok(payload(value)) }) + } + fn decode(&self, _: Payload) -> BoxFuture<'_, Result> { + panic!("remote always misses") + } +} + +#[test] +fn borrowed_only_codecs_keep_working_through_both_apis() { + for registered in [false, true] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let remote = Arc::new(RecordingRemote::default()); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote_arc(remote.clone()) + .build() + .unwrap(); + let encoded = Arc::new(AtomicUsize::new(0)); + let lookup = Lookup::new( + &cache, + Arc::new(BorrowedCodec(encoded.clone())), + registered, + Arc::new(AtomicUsize::new(0)), + ); + executor.block_on(async move { + let request = cache.enable_guard(); + assert_eq!(*lookup.get(request.scope()).await, Value(7)); + }); + assert_eq!(encoded.load(Ordering::SeqCst), 1); + let writes = remote.0.lock(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].payload, payload(&Value(7))); + } +} + +#[cfg(feature = "tokio")] +#[tokio::test(flavor = "current_thread")] +async fn owned_encoders_move_non_clone_values_off_thread_through_both_apis() { + struct BackgroundCodec { + started: tokio::sync::mpsc::UnboundedSender<(std::thread::ThreadId, Arc)>, + release: Arc>>, + } + impl Codec for BackgroundCodec { + fn encode<'a>(&'a self, _: &'a Value) -> BoxFuture<'a, Result> { + panic!("the engine must select the owned hook") + } + fn encode_owned(&self, value: Arc) -> BoxFuture<'_, Result> { + let (started, release) = (self.started.clone(), self.release.clone()); + let job = tokio::task::spawn_blocking(move || -> Result { + started.send((std::thread::current().id(), value.clone()))?; + release.lock().recv_timeout(Duration::from_secs(5))?; + Ok(payload(&value)) + }); + Box::pin(async move { job.await? }) + } + fn decode(&self, _: Payload) -> BoxFuture<'_, Result> { + panic!("remote always misses") + } + } + + for registered in [false, true] { + let remote = Arc::new(RecordingRemote::default()); + let cache = DialCache::builder() + .remote_arc(remote.clone()) + .build() + .unwrap(); + let request = cache.enable_guard(); + let (started, mut started_rx) = tokio::sync::mpsc::unbounded_channel(); + let (release, gate) = std::sync::mpsc::channel(); + let lookup = Lookup::new( + &cache, + Arc::new(BackgroundCodec { + started, + release: Arc::new(Mutex::new(gate)), + }), + registered, + Arc::new(AtomicUsize::new(0)), + ); + let scope = request.scope().clone(); + let pending = tokio::spawn(async move { lookup.get(&scope).await }); + let (worker, observed) = tokio::time::timeout(Duration::from_secs(2), started_rx.recv()) + .await + .unwrap() + .expect("owned encoder started"); + assert_ne!(worker, std::thread::current().id()); + tokio::time::sleep(Duration::from_millis(1)).await; + assert!( + !pending.is_finished(), + "encoding remains gated while the timer progresses" + ); + release.send(()).unwrap(); + let value = pending.await.unwrap(); + assert_eq!(*value, Value(7)); + assert!(Arc::ptr_eq(&value, &observed)); + let writes = remote.0.lock(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].payload, payload(&value)); + } +} + +#[derive(Default)] +struct Events(Mutex>); +impl Observer for Events { + fn observe(&self, event: &Event) { + self.0.lock().push(event.clone()); + } +} + +struct RejectCpu { + step: Arc, + attempts: AtomicUsize, + ran: Arc, +} +impl Runtime for RejectCpu { + fn spawn(&self, task: BoxFuture<'static, ()>) { + self.step.spawn(task); + } + fn sleep(&self, delay: Duration) -> BoxFuture<'static, ()> { + self.step.sleep(delay) + } + fn spawn_blocking(&self, _: Box) -> Result<(), BoxError> { + self.attempts.fetch_add(1, Ordering::SeqCst); + Err("custom CPU admission rejected".into()) + } +} + +#[derive(Clone, Copy)] +enum Failure { + Construction, + Poll, + Error, + Rejection, +} +struct FailingCodec { + mode: Failure, + cpu: Arc, + calls: Arc, +} +impl Codec for FailingCodec { + fn encode<'a>(&'a self, _: &'a Value) -> BoxFuture<'a, Result> { + panic!("borrowed encode must not be selected") + } + fn encode_owned(&self, _: Arc) -> BoxFuture<'_, Result> { + self.calls.fetch_add(1, Ordering::SeqCst); + if matches!(self.mode, Failure::Construction) { + panic!("owned hook construction"); + } + Box::pin(async move { + match self.mode { + Failure::Construction => unreachable!(), + Failure::Poll => panic!("owned hook polling"), + Failure::Error => Err("owned hook error".into()), + Failure::Rejection => { + let ran = self.cpu.ran.clone(); + self.cpu.spawn_blocking(Box::new(move || { + ran.store(true, Ordering::SeqCst); + }))?; + Ok(Payload::text("unexpected admission")) + } + } + }) + } + fn decode(&self, _: Payload) -> BoxFuture<'_, Result> { + panic!("remote always misses") + } +} + +#[test] +fn owned_hook_failures_preserve_source_and_local_publication() { + for registered in [false, true] { + for mode in [ + Failure::Construction, + Failure::Poll, + Failure::Error, + Failure::Rejection, + ] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let remote = Arc::new(RecordingRemote::default()); + let events = Arc::new(Events::default()); + let cpu = Arc::new(RejectCpu { + step: executor.runtime.clone(), + attempts: AtomicUsize::new(0), + ran: Arc::new(AtomicBool::new(false)), + }); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote_arc(remote.clone()) + .observer_arc(events.clone()) + .build() + .unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let encoded = Arc::new(AtomicUsize::new(0)); + let lookup = Lookup::new( + &cache, + Arc::new(FailingCodec { + mode, + cpu: cpu.clone(), + calls: encoded.clone(), + }), + registered, + calls.clone(), + ); + executor.block_on(async move { + let request = cache.enable_guard(); + let first = lookup.get(request.scope()).await; + assert_eq!(*first, Value(7)); + assert!(Arc::ptr_eq(&first, &lookup.get(request.scope()).await)); + }); + assert_eq!(calls.load(Ordering::SeqCst), 1); + assert_eq!(encoded.load(Ordering::SeqCst), 1); + assert!(remote.0.lock().is_empty()); + assert_eq!( + cpu.attempts.load(Ordering::SeqCst), + usize::from(matches!(mode, Failure::Rejection)) + ); + assert!(!cpu.ran.load(Ordering::SeqCst)); + let events = events.0.lock(); + assert!(events.iter().any(|e| matches!( + e, + Event::Error { + error: ErrorKind::SerializationDump, + .. + } + ))); + assert!(events.iter().any(|e| matches!( + e, + Event::Serialization { + operation: SerializationOperation::Dump, + .. + } + ))); + } + } +} From 959fc098d166641bf43be0de7c3e2cfede7f78f8 Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Mon, 21 Sep 2026 20:27:39 -0700 Subject: [PATCH 30/31] fix(rust): guard queued shadow writes and simplify payload paths --- rust/src/execution.rs | 11 +- rust/src/protocol/envelope.rs | 66 ++++++----- rust/src/protocol/text.rs | 64 +---------- rust/src/shadow.rs | 37 +++--- rust/tests/shadow_write_deadline.rs | 172 ++++++++++++++++++++++++++++ 5 files changed, 232 insertions(+), 118 deletions(-) create mode 100644 rust/tests/shadow_write_deadline.rs diff --git a/rust/src/execution.rs b/rust/src/execution.rs index c56336f5..4b327a0f 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -1009,18 +1009,25 @@ impl Execution { }, ttl_ms: ttl, }; - let pending: Settled> = start_pending( + let shadow = shadow.cloned(); + let pending: Settled> = start_pending( self.core.runtime.as_ref(), async move { + // Task admission can precede its first poll. Keep the raw task's + // shadow ownership and check again at the adapter boundary. + if shadow.as_ref().is_some_and(ShadowWork::expired) { + return Ok(false); + } remote .write(request) .await + .map(|()| true) .map_err(|e| Arc::from(e) as SharedError) }, |message| Err(Arc::new(PanicError(message)) as SharedError), ); match pending.wait().await { - Ok(()) => Ok(true), + Ok(dispatched) => Ok(dispatched), Err(error) => { self.error_event(layer, ErrorKind::CacheWrite, false); Err(Box::new(SharedErrorWrapper(error))) diff --git a/rust/src/protocol/envelope.rs b/rust/src/protocol/envelope.rs index c56f192b..7aefdd05 100644 --- a/rust/src/protocol/envelope.rs +++ b/rust/src/protocol/envelope.rs @@ -86,14 +86,17 @@ pub struct CompressionReadResult { pub outcome: Option, } -/// Prefix raw binary output whose first byte would collide with the envelope. -pub fn escape_raw_payload(payload: Payload) -> Payload { - if payload.binary +fn needs_raw_escape(payload: &Payload) -> bool { + payload.binary && payload .bytes .first() .is_some_and(|first| *first <= MARKER_ZSTD_BINARY) - { +} + +/// Prefix raw binary output whose first byte would collide with the envelope. +pub fn escape_raw_payload(payload: Payload) -> Payload { + if needs_raw_escape(&payload) { let mut escaped = Vec::with_capacity(payload.bytes.len() + 1); escaped.push(MARKER_ESCAPED_RAW); escaped.extend_from_slice(&payload.bytes); @@ -125,36 +128,37 @@ pub fn compress_payload( } }; let original_bytes = payload.len(); - let raw = escape_raw_payload(payload.clone()); - let mut result = CompressionWriteResult { + let outcome = if original_bytes < config.threshold_bytes { + CompressionOutcome::BelowThreshold + } else if original_bytes > max_decompressed_bytes { + CompressionOutcome::WriteOverLimit + } else { + let compressed = zstd_compress(&payload.bytes, config.level)?; + let raw_bytes = original_bytes + usize::from(needs_raw_escape(&payload)); + if compressed.len() + 1 < raw_bytes { + let mut marked = Vec::with_capacity(compressed.len() + 1); + marked.push(if payload.binary { + MARKER_ZSTD_BINARY + } else { + MARKER_ZSTD_UTF8 + }); + marked.extend_from_slice(&compressed); + return Ok(CompressionWriteResult { + stored_bytes: marked.len(), + payload: Payload::binary(marked), + outcome: CompressionOutcome::Compressed, + original_bytes, + }); + } + CompressionOutcome::NotSmaller + }; + let raw = escape_raw_payload(payload); + Ok(CompressionWriteResult { stored_bytes: raw.len(), payload: raw, - outcome: CompressionOutcome::BelowThreshold, + outcome, original_bytes, - }; - if original_bytes < config.threshold_bytes { - return Ok(result); - } - if original_bytes > max_decompressed_bytes { - result.outcome = CompressionOutcome::WriteOverLimit; - return Ok(result); - } - let compressed = zstd_compress(&payload.bytes, config.level)?; - if compressed.len() + 1 >= result.stored_bytes { - result.outcome = CompressionOutcome::NotSmaller; - return Ok(result); - } - let mut marked = Vec::with_capacity(compressed.len() + 1); - marked.push(if payload.binary { - MARKER_ZSTD_BINARY - } else { - MARKER_ZSTD_UTF8 - }); - marked.extend_from_slice(&compressed); - result.stored_bytes = marked.len(); - result.payload = Payload::binary(marked); - result.outcome = CompressionOutcome::Compressed; - Ok(result) + }) } /// Reverse of [`compress_payload`], applied to every read. diff --git a/rust/src/protocol/text.rs b/rust/src/protocol/text.rs index 905f523a..8338e1b7 100644 --- a/rust/src/protocol/text.rs +++ b/rust/src/protocol/text.rs @@ -5,75 +5,13 @@ //! the WHATWG UTF-8 decoder with replacement error handling and without BOM //! removal, so every maximal ill-formed subpart becomes exactly one U+FFFD. -/// The UTF-8 encoding of U+FFFD. -const REPLACEMENT: [u8; 3] = [0xEF, 0xBF, 0xBD]; - /// Decode bytes as UTF-8 with one U+FFFD per maximal ill-formed subpart and /// no BOM removal (the WHATWG decoder with replacement error handling). /// /// The result is always valid UTF-8. Well-formed input is returned unchanged, /// so valid text costs one validation pass and one copy. pub fn replacement_utf8(bytes: &[u8]) -> Vec { - if std::str::from_utf8(bytes).is_ok() { - return bytes.to_vec(); - } - let mut out = Vec::with_capacity(bytes.len() + REPLACEMENT.len()); - let mut rest = bytes; - while !rest.is_empty() { - match std::str::from_utf8(rest) { - Ok(_) => { - out.extend_from_slice(rest); - break; - } - Err(error) => { - let valid = error.valid_up_to(); - out.extend_from_slice(&rest[..valid]); - rest = &rest[valid..]; - let consumed = ill_formed_subpart_len(rest); - out.extend_from_slice(&REPLACEMENT); - rest = &rest[consumed..]; - } - } - } - out -} - -/// Length of the maximal ill-formed subpart at the start of `bytes`, whose -/// first byte is known not to begin a well-formed sequence. -/// -/// A lead in `C2..DF`, `E0..EF` or `F0..F4` swallows the continuation bytes -/// that are valid *in their position* (including the narrowed second-byte -/// ranges after `E0`, `ED`, `F0` and `F4`); the first byte that is not valid -/// there is left to be reprocessed as the next lead. Any other first byte is a -/// one-byte subpart. -fn ill_formed_subpart_len(bytes: &[u8]) -> usize { - let lead = bytes[0]; - let wanted = match lead { - 0xC2..=0xDF => 2, - 0xE0..=0xEF => 3, - 0xF0..=0xF4 => 4, - _ => return 1, - }; - let mut consumed = 1; - while consumed < wanted && consumed < bytes.len() { - let byte = bytes[consumed]; - let (lower, upper) = if consumed == 1 { - match lead { - 0xE0 => (0xA0, 0xBF), - 0xED => (0x80, 0x9F), - 0xF0 => (0x90, 0xBF), - 0xF4 => (0x80, 0x8F), - _ => (0x80, 0xBF), - } - } else { - (0x80, 0xBF) - }; - if byte < lower || byte > upper { - break; - } - consumed += 1; - } - consumed + String::from_utf8_lossy(bytes).into_owned().into_bytes() } #[cfg(test)] diff --git a/rust/src/shadow.rs b/rust/src/shadow.rs index faa92ef8..35629a58 100644 --- a/rust/src/shadow.rs +++ b/rust/src/shadow.rs @@ -8,7 +8,6 @@ use std::time::Duration; use futures::future::{select, Either}; use futures::FutureExt; -use parking_lot::Mutex; use crate::deadline::{await_deadline, since}; use crate::engine::Core; @@ -254,33 +253,27 @@ impl Execution { budget: Duration::from_millis(budget_ms), }; let log_slot = slot.clone(); - let reads: Arc>>> = Arc::new(Mutex::new(Vec::new())); let validation: Settled = start_pending( self.core.runtime.as_ref(), { let x = self.clone(); - let reads = reads.clone(); async move { - let verdict = match AssertUnwindSafe(x.clone().validate( - work, - frame, - source, - reads.clone(), - )) - .catch_unwind() - .await - { - Ok(verdict) => verdict, - Err(_) => Verdict::of(ShadowOutcome::Timeout), - }; + let mut reads = Vec::new(); + let verdict = + match AssertUnwindSafe(x.clone().validate(work, frame, source, &mut reads)) + .catch_unwind() + .await + { + Ok(verdict) => verdict, + Err(_) => Verdict::of(ShadowOutcome::Timeout), + }; // Reads keep the slot after a read timeout; owned source, codec and // write work already keeps this operation running until raw completion. // Dropping the slot frees it, whether the reads settled or the // runtime dropped this task first. - let pending = std::mem::take(&mut *reads.lock()); let runtime = x.core.runtime.clone(); runtime.spawn(Box::pin(async move { - for read in pending { + for read in reads { let _ = read.wait().await; } drop(slot); @@ -308,7 +301,7 @@ impl Execution { async fn shadow_read( &self, - reads: &Mutex>>, + reads: &mut Vec>, max_age: bool, retain_future: bool, ) -> RawRead { @@ -317,7 +310,7 @@ impl Execution { labels: self.labels(Layer::RemoteShadow), }); let (bounded, raw) = self.raw_read(); - reads.lock().push(raw); + reads.push(raw); let result = bounded.await; let result = match result { Err(error) => { @@ -362,7 +355,7 @@ impl Execution { work: ShadowWork, mut frame: Option, source: Option>, - reads: Arc>>>, + reads: &mut Vec>, ) -> Verdict { let expired = || work.expired(); if expired() { @@ -371,7 +364,7 @@ impl Execution { let mut fill = false; let mut fence: Option = None; if source.is_some() { - let read = match self.shadow_read(&reads, true, false).await { + let read = match self.shadow_read(reads, true, false).await { Ok(read) => read, Err(_) => return Verdict::of(ShadowOutcome::RedisError), }; @@ -462,7 +455,7 @@ impl Execution { compared: None, }; } - let confirmation = match self.shadow_read(&reads, false, true).await { + let confirmation = match self.shadow_read(reads, false, true).await { Ok(confirmation) => confirmation, Err(_) => return Verdict::of(ShadowOutcome::ConfirmationError), }; diff --git a/rust/tests/shadow_write_deadline.rs b/rust/tests/shadow_write_deadline.rs new file mode 100644 index 00000000..33de6875 --- /dev/null +++ b/rust/tests/shadow_write_deadline.rs @@ -0,0 +1,172 @@ +//! A prepared shadow fill may wait in the runtime before its adapter is invoked. +use std::future::ready; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use dialcache::observe::{Layer, ShadowOutcome}; +use dialcache::testing::{StepRuntime, TestExecutor, VirtualClock, WALL_EPOCH_MS}; +use dialcache::{ + BoxError, Clock, DialCache, Event, Identity, InvalidateRequest, MissReason, Observer, + Operation, Policy, ReadContext, ReadRequest, ReadResult, Remote, Runtime, ShadowPolicy, + SourceBudget, WriteRequest, +}; +use futures::future::BoxFuture; +use parking_lot::Mutex; + +struct GatedRuntime { + step: Arc, + hold: AtomicBool, + pending: Mutex>>, +} + +impl Runtime for GatedRuntime { + fn spawn(&self, task: BoxFuture<'static, ()>) { + if self.hold.load(Ordering::SeqCst) { + self.pending.lock().push(task); + } else { + self.step.spawn(task); + } + } + + fn defer(&self, task: BoxFuture<'static, ()>) { + self.step.defer(task); + } + + fn sleep(&self, delay: Duration) -> BoxFuture<'static, ()> { + self.step.sleep(delay) + } +} + +impl GatedRuntime { + fn release(&self) { + self.hold.store(false, Ordering::SeqCst); + for task in std::mem::take(&mut *self.pending.lock()) { + self.step.spawn(task); + } + } +} + +struct RecordingRemote { + clock: Arc, + writes: Mutex>, +} + +impl Remote for RecordingRemote { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(ready(Ok(ReadResult::miss(MissReason::ValueAbsent)))) + } + + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + self.writes.lock().push(self.clock.elapsed()); + Box::pin(ready(Ok(()))) + } + + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + Box::pin(ready(Ok(()))) + } +} + +struct Events { + runtime: Arc, + outcomes: Mutex>, +} + +impl Observer for Events { + fn observes_shadow_outcomes(&self) -> bool { + true + } + + fn observe(&self, event: &Event) { + match event { + Event::StoredSize { labels, .. } if labels.layer == Layer::RemoteShadow => { + self.runtime.hold.store(true, Ordering::SeqCst); + } + Event::ShadowValidation { outcome, .. } => self.outcomes.lock().push(*outcome), + _ => {} + } + } +} + +fn invoke(executor: &mut TestExecutor, cache: &DialCache) { + let cache = cache.clone(); + let value = executor.block_on(async move { + let request = cache.enable_guard(); + cache + .get_or_load( + request.scope(), + Operation::::new(Identity::new("thing", "one", "QueuedWrite")) + .policy( + Policy::default() + .remote_ttl_sec(60) + .remote_ramp(0.0) + .shadow(ShadowPolicy { + ramp: Some(100.0), + log_mismatches: None, + }), + ) + .budget(SourceBudget::Millis(5)), + |_| ready(Ok(7)), + ) + .await + .expect("shadow scheduling must preserve source success") + }); + assert_eq!(*value, 7); +} + +#[test] +fn shadow_writes_start_only_before_the_deadline_even_when_task_starts_are_delayed() { + for (delay, deliver) in [(0, false), (5, false), (5, true), (10, true)] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let runtime = Arc::new(GatedRuntime { + step: executor.runtime.clone(), + hold: AtomicBool::new(false), + pending: Mutex::new(Vec::new()), + }); + let remote = Arc::new(RecordingRemote { + clock: executor.clock.clone(), + writes: Mutex::new(Vec::new()), + }); + let events = Arc::new(Events { + runtime: runtime.clone(), + outcomes: Mutex::new(Vec::new()), + }); + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(runtime.clone()) + .remote_arc(remote.clone()) + .observer_arc(events.clone()) + .shadow_max_in_flight(1) + .build() + .unwrap(); + invoke(&mut executor, &cache); + executor.advance(delay, deliver); + runtime.release(); + executor.drain(); + + let writes = remote.writes.lock().clone(); + assert!( + writes.iter().all(|at| *at < Duration::from_millis(5)), + "adapter called after shadow deadline: {writes:?}, delay={delay}, timers={deliver}" + ); + if delay == 0 { + assert_eq!(writes.len(), 1, "control must actually fill"); + } + assert_eq!( + *events.outcomes.lock(), + vec![if writes.is_empty() { + ShadowOutcome::Timeout + } else { + ShadowOutcome::Filled + }] + ); + + // The completed/skipped raw write must release the only shadow slot. + let before = writes.len(); + invoke(&mut executor, &cache); + runtime.release(); + executor.drain(); + assert_eq!(remote.writes.lock().len(), before + 1); + assert_eq!(events.outcomes.lock().last(), Some(&ShadowOutcome::Filled)); + } +} From 22e82f9c7b5b937aeea3a52d531f4a77292a560b Mon Sep 17 00:00:00 2001 From: Lev Neiman Date: Mon, 21 Sep 2026 21:07:24 -0700 Subject: [PATCH 31/31] fix(rust): share read snapshots and redact recovery diagnostics --- formal/rust-mutations.json | 4 +- rust/Cargo.lock | 12 +++ rust/Cargo.toml | 1 + rust/README.md | 4 + rust/src/execution.rs | 61 +++++++++++--- rust/src/observe.rs | 24 ++++-- rust/src/read_snapshot_tests.rs | 137 +++++++++++++++++++++++++++++++ rust/src/shadow.rs | 24 +++--- rust/tests/conformance.rs | 3 + rust/tests/formal/digest.rs | 6 ++ rust/tests/formal/fixtures.rs | 83 +------------------ rust/tests/formal/witness.rs | 81 +----------------- rust/tests/harness_infra.rs | 3 + rust/tests/protocol_frames.rs | 3 + rust/tests/protocol_keys.rs | 3 + rust/tests/recovery_logging.rs | 125 ++++++++++++++++++++++++++++ rust/tests/redis_integration.rs | 3 + rust/tests/settlement_control.rs | 3 + 18 files changed, 388 insertions(+), 192 deletions(-) create mode 100644 rust/src/read_snapshot_tests.rs create mode 100644 rust/tests/formal/digest.rs create mode 100644 rust/tests/recovery_logging.rs diff --git a/formal/rust-mutations.json b/formal/rust-mutations.json index b425b757..89909ac0 100644 --- a/formal/rust-mutations.json +++ b/formal/rust-mutations.json @@ -147,8 +147,8 @@ "edits": [ { "path": "rust/src/shadow.rs", - "before": " ReadResult::Hit(confirmed) if confirmed.payload.bytes == frame.payload.bytes => {}", - "after": " ReadResult::Hit(_) => {}" + "before": " ReadSnapshot::Hit(confirmed) if confirmed.payload.bytes == frame.payload.bytes => {}", + "after": " ReadSnapshot::Hit(_) => {}" } ], "requiredDetections": [ diff --git a/rust/Cargo.lock b/rust/Cargo.lock index be3c45d8..3f1f8179 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -159,6 +159,7 @@ dependencies = [ "serde", "serde_json", "sha1", + "sha2", "slab", "thiserror 2.0.20", "tokio", @@ -852,6 +853,17 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 80959b59..4534e9f2 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -37,6 +37,7 @@ hex = { version = "0.4", optional = true } prometheus = { version = "0.14", optional = true } [dev-dependencies] +sha2 = "0.11" dialcache = { path = ".", features = ["test-util"] } tokio = { version = "1", features = ["full"] } hex = "0.4" diff --git a/rust/README.md b/rust/README.md index d1db4e22..ec81b60f 100644 --- a/rust/README.md +++ b/rust/README.md @@ -214,6 +214,10 @@ synchronous, so large values can occupy an async worker. validation exists only to be observed, so a job is admitted only when the observer opts in through `observes_shadow_outcomes`; the bundled exporters do. `Logger` receives structured `LogEvent`s and defaults to the `log` facade. +Default stale-recovery decode warnings omit error text that could contain cached +values; JSON errors retain their category and line/column location. A custom +`Logger` can inspect the original error when application-controlled details are +needed. Mismatch logging is opt-in, confirmed, bounded, and previews values through the operation's `preview` (JSON for serde values). Default JSON previews retain only an 8 KiB prefix while checking the entire serialization for errors. Preview diff --git a/rust/src/execution.rs b/rust/src/execution.rs index 4b327a0f..fec0c8f9 100644 --- a/rust/src/execution.rs +++ b/rust/src/execution.rs @@ -30,8 +30,43 @@ use crate::remote::{Frame, ReadContext, ReadRequest, ReadResult, WriteRequest}; use crate::scope::{Owner, Scope}; use crate::shadow::ShadowWork; -/// The raw outcome of one adapter read. -pub(crate) type RawRead = Result; +/// An immutable acquired frame shared by readers of the same settled result. +/// The public adapter result stays owned; only internal observations share it. +#[derive(Debug, Clone)] +pub(crate) enum ReadSnapshot { + Hit(Arc), + Miss { + reason: MissReason, + observed_watermark_ms: Option, + }, +} + +impl ReadSnapshot { + pub(crate) fn miss(reason: MissReason) -> Self { + Self::Miss { + reason, + observed_watermark_ms: None, + } + } +} + +impl From for ReadSnapshot { + fn from(read: ReadResult) -> Self { + match read { + ReadResult::Hit(frame) => Self::Hit(Arc::new(frame)), + ReadResult::Miss { + reason, + observed_watermark_ms, + } => Self::Miss { + reason, + observed_watermark_ms, + }, + } + } +} + +/// The normalized, shared outcome of one raw adapter read. +pub(crate) type RawRead = Result; #[derive(Debug, thiserror::Error)] #[error("DialCache callback panicked: {0}")] @@ -113,11 +148,11 @@ pub(crate) struct Execution { pub(crate) enum RemoteValue { Hit { value: StoredValue, - frame: Frame, + frame: Arc, }, /// A valid stale frame retained only as a recovery candidate. Retained { - frame: Frame, + frame: Arc, }, Miss { fence: Option, @@ -617,7 +652,7 @@ impl Execution { let recoverable = !matches!(remote, RemoteValue::Error | RemoteValue::DecodeError); if recoverable && p.stale_on_error_max_age_ms > 0 && self.can_recover(&error) { let frame = match &remote { - RemoteValue::Retained { frame } => Some(frame), + RemoteValue::Retained { frame } => Some(frame.as_ref()), _ => None, }; if let Some(value) = self.recover(frame).await { @@ -679,21 +714,22 @@ impl Execution { let clock = self.core.clock.clone(); let runtime = self.core.runtime.clone(); let start = clock.elapsed(); + let tracked = self.identity.tracked; let raw: Settled = start_pending( self.core.runtime.as_ref(), async move { remote .read(request, context) .await + .map(|read| ReadSnapshot::from(normalize_read_result(read, tracked))) .map_err(|e| Arc::from(e) as SharedError) }, |message| Err(Arc::new(PanicError(message)) as SharedError), ); - let tracked = self.identity.tracked; let use_case = self.labels.use_case.to_string(); let waited = raw.clone(); let bounded = Box::pin(async move { - let result = await_deadline( + await_deadline( clock.as_ref(), runtime.as_ref(), &waited, @@ -707,8 +743,7 @@ impl Execution { }) as SharedError) }, ) - .await; - result.map(|read| normalize_read_result(read, tracked)) + .await }); (bounded, raw) } @@ -750,7 +785,7 @@ impl Execution { self.error_event(Layer::Remote, kind, false); RemoteValue::Error } - Ok(ReadResult::Miss { + Ok(ReadSnapshot::Miss { reason, observed_watermark_ms, }) => { @@ -762,7 +797,7 @@ impl Execution { fence: observed_watermark_ms, } } - Ok(ReadResult::Hit(frame)) => { + Ok(ReadSnapshot::Hit(frame)) => { let (age, valid) = self.frame_age(&frame, Layer::Remote); if !valid { self.emit(Event::Miss { @@ -1100,3 +1135,7 @@ impl Execution { Some(value) } } + +#[cfg(test)] +#[path = "read_snapshot_tests.rs"] +mod read_snapshot_tests; diff --git a/rust/src/observe.rs b/rust/src/observe.rs index 9364b0c8..2d38e062 100644 --- a/rust/src/observe.rs +++ b/rust/src/observe.rs @@ -477,7 +477,8 @@ pub enum LogEvent { /// The stale-recovery classifier panicked; recovery was denied. RecoveryPredicateFailed(BoxError), /// Decoding the retained stale frame failed during recovery; the source - /// error was returned. + /// error was returned. Default formatting omits value-bearing error text; + /// custom loggers can inspect the original error here. RecoveryDecodeFailed(BoxError), /// A shadow job's clean-miss fill failed while preparing or writing the payload. ShadowFillFailed(BoxError), @@ -518,10 +519,23 @@ impl fmt::Display for LogEvent { "DialCache stale recovery predicate failed; recovery was denied: {e}" ) } - LogEvent::RecoveryDecodeFailed(e) => write!( - f, - "Error using retained Redis value during stale recovery: {e}" - ), + LogEvent::RecoveryDecodeFailed(e) => { + write!( + f, + "Error using retained Redis value during stale recovery: " + )?; + if let Some(json) = e.downcast_ref::() { + write!( + f, + "JSON {:?} at line {} column {}", + json.classify(), + json.line(), + json.column() + ) + } else { + write!(f, "decoding failed") + } + } LogEvent::ShadowFillFailed(e) => { write!(f, "Error populating Redis from DialCache shadow work: {e}") } diff --git a/rust/src/read_snapshot_tests.rs b/rust/src/read_snapshot_tests.rs new file mode 100644 index 00000000..6af64831 --- /dev/null +++ b/rust/src/read_snapshot_tests.rs @@ -0,0 +1,137 @@ +//! Native allocation and codec-ownership boundaries for acquired remote frames. +use super::*; +use crate::operation::{downcast_value, erase_load, Operation}; +use crate::testing::{TestExecutor, WALL_EPOCH_MS}; +use crate::{Codec, DialCache, InvalidateRequest, Payload, Policy, Remote}; +use futures::channel::oneshot; +use parking_lot::Mutex; + +struct OneRead(Mutex>); +impl Remote for OneRead { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(std::future::ready(Ok(ReadResult::Hit( + self.0.lock().take().unwrap(), + )))) + } + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + panic!("this test only reads") + } + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + panic!("this test only reads") + } +} +struct GatedCodec { + gate: Mutex>>, + input: Settled, +} +impl Codec for GatedCodec { + fn encode(&self, _: &u64) -> BoxFuture<'_, Result> { + panic!("this test only decodes") + } + fn decode(&self, mut payload: Payload) -> BoxFuture<'_, Result> { + let gate = self.gate.lock().take().unwrap(); + Box::pin(async move { + self.input.settle(payload.bytes.as_ptr() as usize); + payload.bytes[0] = 99; + gate.await?; + Ok(payload.len() as u64) + }) + } +} +fn hit(result: RawRead) -> Arc { + match result.unwrap() { + ReadSnapshot::Hit(frame) => frame, + ReadSnapshot::Miss { .. } => panic!("expected a hit"), + } +} + +#[test] +fn acquired_frames_share_storage_while_a_gated_codec_owns_an_independent_input() { + for ready_first in [false, true] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let payload = Payload::binary(vec![42; 128 * 1024]); + let original_buffer = payload.bytes.as_ptr() as usize; + let cache = DialCache::builder() + .clock_arc(executor.clock.clone()) + .runtime_arc(executor.runtime.clone()) + .remote(OneRead(Mutex::new(Some(Frame { + created_at_ms: WALL_EPOCH_MS as u64, + payload, + })))) + .build() + .unwrap(); + let (release, gate) = oneshot::channel(); + let codec = Arc::new(GatedCodec { + gate: Mutex::new(Some(gate)), + input: Settled::new(), + }); + let (identity, metadata) = Operation::with_codec( + Identity::new("thing", "one", "ReadSnapshot"), + codec.clone(), + |a, b| a == b, + ) + .policy(Policy::default().remote_ttl_sec(60)) + .erase(); + let keys = identity.keys().unwrap(); + let policy = resolve_policy( + &metadata.policy, + None, + &keys.logical, + PolicyDefaults { + remote_read_timeout_ms: 100, + }, + ) + .unwrap(); + let request = cache.enable_guard(); + let execution = Arc::new(Execution { + core: cache.core.clone(), + scope: request.scope().clone(), + op: Arc::new(ErasedOperation { + identity: identity.clone(), + identity_provider: None, + metadata, + load: erase_load(|_| async { Ok(0u64) }), + }), + labels: crate::engine::outcome_labels(&identity), + identity, + keys, + policy, + }); + let (bounded, raw) = execution.raw_read(); + if ready_first { + executor.drain(); + } + // The two schedules cover the deadline cell's ready peek and pending wait. + let frame = hit(executor.block_on(bounded)); + let observed = hit(raw.peek().unwrap()); + assert!(Arc::ptr_eq(&frame, &observed)); + assert_eq!(frame.payload.bytes.as_ptr() as usize, original_buffer); + let weak = Arc::downgrade(&frame); + let (done, receive) = oneshot::channel(); + let decoding = execution.clone(); + let decoding_frame = frame.clone(); + executor.spawn(async move { + let result = decoding.decode(&decoding_frame, Layer::Remote, None).await; + done.send(result).unwrap(); + }); + executor.drain(); + assert_ne!(codec.input.peek().expect("codec started"), original_buffer); + assert!( + frame.payload.bytes.iter().all(|byte| *byte == 42), + "codec mutation leaked into the retained snapshot" + ); + let cleanup = hit(executor.block_on(async move { raw.wait().await })); + assert!( + Arc::ptr_eq(&frame, &cleanup), + "cleanup must observe the shared allocation" + ); + release.send(()).unwrap(); + let value = executor.block_on(async move { receive.await.unwrap().unwrap() }); + assert_eq!(*downcast_value::(value).unwrap(), 128 * 1024); + drop((frame, observed, cleanup)); + assert!( + weak.upgrade().is_none(), + "completed observers retained the frame" + ); + } +} diff --git a/rust/src/shadow.rs b/rust/src/shadow.rs index 35629a58..6c945742 100644 --- a/rust/src/shadow.rs +++ b/rust/src/shadow.rs @@ -12,7 +12,7 @@ use futures::FutureExt; use crate::deadline::{await_deadline, since}; use crate::engine::Core; use crate::error::Error; -use crate::execution::{call_load, Execution, RawRead}; +use crate::execution::{call_load, Execution, RawRead, ReadSnapshot}; use crate::flight::{start_pending, yield_deferred, Settled, ValueResult}; use crate::limits::DEFAULT_FALLBACK_TIMEOUT_MS; use crate::local::StoredValue; @@ -20,7 +20,7 @@ use crate::observe::{ ErrorKind, Event, Layer, LogEvent, MissReason, ShadowMismatchDetails, ShadowOutcome, }; use crate::preview::{preview_key, preview_value}; -use crate::remote::{Frame, ReadResult}; +use crate::remote::Frame; /// One admitted shadow job holding an instance slot. /// One held entry of the instance-wide shadow table. Dropping it frees the @@ -134,7 +134,7 @@ impl Execution { /// instance capacity allow it. pub(crate) fn schedule_shadow( self: &Arc, - frame: Option, + frame: Option>, source: Option>, started: Duration, ) { @@ -228,7 +228,7 @@ impl Execution { async fn run_shadow( self: Arc, flight: Arc, - frame: Option, + frame: Option>, source: Option>, started: Duration, slot: ShadowSlot, @@ -326,15 +326,15 @@ impl Execution { Err(error) } Ok(mut read) => { - if let ReadResult::Hit(frame) = &read { + if let ReadSnapshot::Hit(frame) = &read { let (age, valid) = self.frame_age(frame, Layer::RemoteShadow); if !valid && !(retain_future && age < 0) { - read = ReadResult::miss(MissReason::Unclassified); + read = ReadSnapshot::miss(MissReason::Unclassified); } else if max_age && valid && age as u64 >= self.policy.remote.ttl_ms { - read = ReadResult::miss(MissReason::Expired); + read = ReadSnapshot::miss(MissReason::Expired); } } - if let ReadResult::Miss { reason, .. } = &read { + if let ReadSnapshot::Miss { reason, .. } = &read { self.emit(Event::Miss { labels: self.labels(Layer::RemoteShadow), reason: *reason, @@ -353,7 +353,7 @@ impl Execution { async fn validate( self: Arc, work: ShadowWork, - mut frame: Option, + mut frame: Option>, source: Option>, reads: &mut Vec>, ) -> Verdict { @@ -372,14 +372,14 @@ impl Execution { return Verdict::of(ShadowOutcome::Timeout); } match read { - ReadResult::Miss { + ReadSnapshot::Miss { observed_watermark_ms, .. } => { fill = true; fence = observed_watermark_ms; } - ReadResult::Hit(read_frame) => frame = Some(read_frame), + ReadSnapshot::Hit(read_frame) => frame = Some(read_frame), } } let value: ValueResult = match &source { @@ -463,7 +463,7 @@ impl Execution { return Verdict::of(ShadowOutcome::Timeout); } match confirmation { - ReadResult::Hit(confirmed) if confirmed.payload.bytes == frame.payload.bytes => {} + ReadSnapshot::Hit(confirmed) if confirmed.payload.bytes == frame.payload.bytes => {} _ => return Verdict::of(ShadowOutcome::Superseded), } let age = self diff --git a/rust/tests/conformance.rs b/rust/tests/conformance.rs index 86764925..39845539 100644 --- a/rust/tests/conformance.rs +++ b/rust/tests/conformance.rs @@ -11,6 +11,9 @@ #![allow(dead_code)] +#[path = "formal/digest.rs"] +mod digest; + #[path = "formal/fixtures.rs"] mod fixtures; mod formal; diff --git a/rust/tests/formal/digest.rs b/rust/tests/formal/digest.rs new file mode 100644 index 00000000..83e3466d --- /dev/null +++ b/rust/tests/formal/digest.rs @@ -0,0 +1,6 @@ +//! Shared test-only source and artifact fingerprints. +use sha2::{Digest, Sha256}; + +pub fn sha256_hex(bytes: &[u8]) -> String { + hex::encode(Sha256::digest(bytes)) +} diff --git a/rust/tests/formal/fixtures.rs b/rust/tests/formal/fixtures.rs index de42b8e5..f969a8e4 100644 --- a/rust/tests/formal/fixtures.rs +++ b/rust/tests/formal/fixtures.rs @@ -108,88 +108,7 @@ fn unicode_escape_at(bytes: &[u8], index: usize) -> Option { u16::from_str_radix(hex, 16).ok() } -/// Hex-encoded SHA-256 of `bytes`. -pub fn sha256_hex(bytes: &[u8]) -> String { - sha256(bytes) - .iter() - .fold(String::with_capacity(64), |mut hex, byte| { - use std::fmt::Write as _; - let _ = write!(hex, "{byte:02x}"); - hex - }) -} - -/// SHA-256 (FIPS 180-4) without a dependency; the fixture reader only needs -/// to fingerprint a handful of small source files. -pub fn sha256(message: &[u8]) -> [u8; 32] { - const K: [u32; 64] = [ - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, - 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, - 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, - 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, - 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, - 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, - 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, - 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, - 0xc67178f2, - ]; - let mut state: [u32; 8] = [ - 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, - 0x5be0cd19, - ]; - - let mut padded = message.to_vec(); - padded.push(0x80); - while padded.len() % 64 != 56 { - padded.push(0); - } - padded.extend_from_slice(&((message.len() as u64) * 8).to_be_bytes()); - - for block in padded.chunks_exact(64) { - let mut w = [0u32; 64]; - for (i, word) in block.chunks_exact(4).enumerate() { - w[i] = u32::from_be_bytes([word[0], word[1], word[2], word[3]]); - } - for i in 16..64 { - let s0 = w[i - 15].rotate_right(7) ^ w[i - 15].rotate_right(18) ^ (w[i - 15] >> 3); - let s1 = w[i - 2].rotate_right(17) ^ w[i - 2].rotate_right(19) ^ (w[i - 2] >> 10); - w[i] = w[i - 16] - .wrapping_add(s0) - .wrapping_add(w[i - 7]) - .wrapping_add(s1); - } - let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut h] = state; - for i in 0..64 { - let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); - let ch = (e & f) ^ (!e & g); - let t1 = h - .wrapping_add(s1) - .wrapping_add(ch) - .wrapping_add(K[i]) - .wrapping_add(w[i]); - let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); - let maj = (a & b) ^ (a & c) ^ (b & c); - let t2 = s0.wrapping_add(maj); - h = g; - g = f; - f = e; - e = d.wrapping_add(t1); - d = c; - c = b; - b = a; - a = t1.wrapping_add(t2); - } - for (slot, value) in state.iter_mut().zip([a, b, c, d, e, f, g, h]) { - *slot = slot.wrapping_add(value); - } - } - let mut digest = [0u8; 32]; - for (chunk, word) in digest.chunks_exact_mut(4).zip(state) { - chunk.copy_from_slice(&word.to_be_bytes()); - } - digest -} +pub use crate::digest::sha256_hex; /// Every generated protocol vector group registered in `formal/execution.json`, /// keyed by field name, after validating each artifact's schema, provenance diff --git a/rust/tests/formal/witness.rs b/rust/tests/formal/witness.rs index 24c1e4be..2ce08c83 100644 --- a/rust/tests/formal/witness.rs +++ b/rust/tests/formal/witness.rs @@ -19,13 +19,7 @@ pub fn file_sha256(path: &Path) -> Result { Ok(sha256_hex(&raw)) } -/// Hex SHA-256 of `data`. -pub fn sha256_hex(data: &[u8]) -> String { - sha256::digest(data) - .iter() - .map(|byte| format!("{byte:02x}")) - .collect() -} +pub use crate::digest::sha256_hex; fn null_default<'de, D: Deserializer<'de>, T: Default + Deserialize<'de>>( deserializer: D, @@ -356,76 +350,3 @@ fn walk(root: &Path, directory: &Path, out: &mut Vec) -> Result<(), Stri } Ok(()) } - -/// Self-contained SHA-256 (FIPS 180-4), kept private so the harness needs no -/// extra dependency for file fingerprints. -mod sha256 { - const K: [u32; 64] = [ - 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, 0x59f111f1, 0x923f82a4, - 0xab1c5ed5, 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, - 0x9bdc06a7, 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, 0x2de92c6f, - 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, - 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, - 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, 0xa2bfe8a1, 0xa81a664b, - 0xc24b8b70, 0xc76c51a3, 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, - 0x1e376c08, 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, - 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, 0x90befffa, 0xa4506ceb, 0xbef9a3f7, - 0xc67178f2, - ]; - - pub fn digest(data: &[u8]) -> [u8; 32] { - let mut state: [u32; 8] = [ - 0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, 0x510e527f, 0x9b05688c, 0x1f83d9ab, - 0x5be0cd19, - ]; - let mut message = data.to_vec(); - message.push(0x80); - while message.len() % 64 != 56 { - message.push(0); - } - message.extend_from_slice(&((data.len() as u64) * 8).to_be_bytes()); - for block in message.chunks_exact(64) { - let mut w = [0u32; 64]; - for (i, word) in block.chunks_exact(4).enumerate() { - w[i] = u32::from_be_bytes([word[0], word[1], word[2], word[3]]); - } - for i in 16..64 { - let s0 = w[i - 15].rotate_right(7) ^ w[i - 15].rotate_right(18) ^ (w[i - 15] >> 3); - let s1 = w[i - 2].rotate_right(17) ^ w[i - 2].rotate_right(19) ^ (w[i - 2] >> 10); - w[i] = w[i - 16] - .wrapping_add(s0) - .wrapping_add(w[i - 7]) - .wrapping_add(s1); - } - let [mut a, mut b, mut c, mut d, mut e, mut f, mut g, mut h] = state; - for i in 0..64 { - let s1 = e.rotate_right(6) ^ e.rotate_right(11) ^ e.rotate_right(25); - let ch = (e & f) ^ (!e & g); - let t1 = h - .wrapping_add(s1) - .wrapping_add(ch) - .wrapping_add(K[i]) - .wrapping_add(w[i]); - let s0 = a.rotate_right(2) ^ a.rotate_right(13) ^ a.rotate_right(22); - let maj = (a & b) ^ (a & c) ^ (b & c); - let t2 = s0.wrapping_add(maj); - h = g; - g = f; - f = e; - e = d.wrapping_add(t1); - d = c; - c = b; - b = a; - a = t1.wrapping_add(t2); - } - for (slot, value) in state.iter_mut().zip([a, b, c, d, e, f, g, h]) { - *slot = slot.wrapping_add(value); - } - } - let mut out = [0u8; 32]; - for (chunk, word) in out.chunks_exact_mut(4).zip(state) { - chunk.copy_from_slice(&word.to_be_bytes()); - } - out - } -} diff --git a/rust/tests/harness_infra.rs b/rust/tests/harness_infra.rs index d2a8d361..b8837195 100644 --- a/rust/tests/harness_infra.rs +++ b/rust/tests/harness_infra.rs @@ -7,6 +7,9 @@ //! `formal::transport`). Prepend `/opt/homebrew/opt/node@24/bin` to `PATH` //! when the default toolchain is another major version. +#[path = "formal/digest.rs"] +mod digest; + mod formal; use formal::inventory::{ diff --git a/rust/tests/protocol_frames.rs b/rust/tests/protocol_frames.rs index 386f4406..5f913f0d 100644 --- a/rust/tests/protocol_frames.rs +++ b/rust/tests/protocol_frames.rs @@ -2,6 +2,9 @@ //! portable protocol corpus (`formal/protocol-vectors.json` plus the //! Quint-generated frame and envelope artifacts). +#[path = "formal/digest.rs"] +mod digest; + // TODO(integrator): switch to `formal/fixtures.rs` and delete `formal/fixtures_tmp.rs`. #[path = "formal/fixtures.rs"] mod fixtures; diff --git a/rust/tests/protocol_keys.rs b/rust/tests/protocol_keys.rs index 3914b3f1..5afd540e 100644 --- a/rust/tests/protocol_keys.rs +++ b/rust/tests/protocol_keys.rs @@ -4,6 +4,9 @@ //! //! Set `DIALCACHE_PROTOCOL_CORPUS=fixed|generated` to narrow the corpus. +#[path = "formal/digest.rs"] +mod digest; + #[path = "formal/fixtures.rs"] mod fixtures; #[path = "formal/key_vectors.rs"] diff --git a/rust/tests/recovery_logging.rs b/rust/tests/recovery_logging.rs new file mode 100644 index 00000000..1b0e43e2 --- /dev/null +++ b/rust/tests/recovery_logging.rs @@ -0,0 +1,125 @@ +//! Recovery diagnostics must not disclose retained values through the default logger. +use dialcache::observe::LogFacadeLogger; +use dialcache::testing::{TestExecutor, WALL_EPOCH_MS}; +use dialcache::{ + BoxError, DialCache, Frame, Identity, InvalidateRequest, LogEvent, Logger, Operation, Payload, + Policy, ReadContext, ReadRequest, ReadResult, Remote, WriteRequest, +}; +use futures::future::BoxFuture; +use std::future::ready; +use std::sync::{Arc, Mutex}; + +const SECRET: &str = "private-retained-token-7f61"; +struct Capture(Mutex>); +impl log::Log for Capture { + fn enabled(&self, _: &log::Metadata<'_>) -> bool { + true + } + fn log(&self, record: &log::Record<'_>) { + self.0.lock().unwrap().push(record.args().to_string()); + } + fn flush(&self) {} +} +static CAPTURE: Capture = Capture(Mutex::new(Vec::new())); + +#[derive(Default)] +struct DetailedLogger(Mutex>); +impl Logger for DetailedLogger { + fn log(&self, event: &LogEvent) { + if let LogEvent::RecoveryDecodeFailed(error) = event { + let error = error + .downcast_ref::() + .expect("original codec error"); + self.0.lock().unwrap().push(error.to_string()); + } + } +} +struct RetainedString; +impl Remote for RetainedString { + fn read(&self, _: ReadRequest, _: ReadContext) -> BoxFuture<'_, Result> { + Box::pin(ready(Ok(ReadResult::Hit(Frame { + created_at_ms: WALL_EPOCH_MS as u64 - 1500, + payload: Payload::text(serde_json::to_string(SECRET).unwrap()), + })))) + } + fn write(&self, _: WriteRequest) -> BoxFuture<'_, Result<(), BoxError>> { + panic!("failed recovery must not write a value") + } + fn invalidate(&self, _: InvalidateRequest) -> BoxFuture<'_, Result<(), BoxError>> { + panic!("no invalidation requested") + } +} +#[derive(Debug, thiserror::Error)] +#[error("source unavailable")] +struct SourceFailure(Arc<()>); + +#[test] +fn recovery_warning_omits_cached_values_and_preserves_structured_errors() { + log::set_logger(&CAPTURE).unwrap(); + log::set_max_level(log::LevelFilter::Warn); + for custom_logger in [false, true] { + let mut executor = TestExecutor::new(WALL_EPOCH_MS); + let detailed = Arc::new(DetailedLogger::default()); + let mut builder = DialCache::builder() + .runtime_arc(executor.runtime.clone()) + .clock_arc(executor.clock.clone()) + .remote(RetainedString); + if custom_logger { + builder = builder.logger_arc(detailed.clone()); + } + let cache = builder.build().unwrap(); + let marker = Arc::new(()); + let source_marker = marker.clone(); + let error = executor.block_on(async move { + let request = cache.enable_guard(); + cache + .get_or_load( + request.scope(), + Operation::::new(Identity::new("thing", "one", "PrivateRecovery")) + .policy( + Policy::default() + .remote_ttl_sec(1) + .stale_on_error_max_age_sec(10), + ) + .should_recover(|_| true), + move |_| { + ready(Err( + Box::new(SourceFailure(source_marker.clone())) as BoxError + )) + }, + ) + .await + .expect_err("incompatible retained value must preserve source failure") + }); + let original = error + .source_error() + .unwrap() + .downcast_ref::() + .unwrap(); + assert!(Arc::ptr_eq(&original.0, &marker)); + if custom_logger { + let details = detailed.0.lock().unwrap(); + assert_eq!(details.len(), 1); + assert!( + details[0].contains(SECRET), + "explicit logger retains original diagnostic" + ); + } else { + let messages = CAPTURE.0.lock().unwrap(); + assert_eq!(messages.len(), 1, "actual default warning must be emitted"); + assert!( + !messages[0].contains(SECRET), + "cached value leaked: {}", + messages[0] + ); + assert!(messages[0].contains("JSON Data")); + assert!(messages[0].contains("line 1")); + } + } + // Arbitrary codec errors may also contain values: the default representation + // cannot trust their Display implementation even without JSON metadata. + LogFacadeLogger.log(&LogEvent::RecoveryDecodeFailed(SECRET.into())); + let messages = CAPTURE.0.lock().unwrap(); + assert_eq!(messages.len(), 2); + assert!(!messages[1].contains(SECRET)); +} diff --git a/rust/tests/redis_integration.rs b/rust/tests/redis_integration.rs index ff28a326..2a29b611 100644 --- a/rust/tests/redis_integration.rs +++ b/rust/tests/redis_integration.rs @@ -31,6 +31,9 @@ #![cfg(feature = "redis")] +#[path = "formal/digest.rs"] +mod digest; + mod formal; #[path = "formal/invalidation_vectors.rs"] mod invalidation_vectors; diff --git a/rust/tests/settlement_control.rs b/rust/tests/settlement_control.rs index 297b033b..a451c512 100644 --- a/rust/tests/settlement_control.rs +++ b/rust/tests/settlement_control.rs @@ -1,6 +1,9 @@ //! The no-settle control: skipping the driver drain must fail the settlement //! contract of every behavior smoke history before observation comparison. +#[path = "formal/digest.rs"] +mod digest; + mod formal; use formal::driver::{install_panic_hook, Driver};