diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 21bfceb..01e26ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,12 +90,64 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + if: matrix.os == 'macos-latest' + with: + node-version: 22 - uses: actions-rust-lang/setup-rust-toolchain@9d7e65c320fdb52dcd45ffaa68deb6c02c8754d9 # v1.12.0 with: toolchain: stable rustflags: "" - name: Run tests run: cargo test --workspace --verbose + - name: Run multicast mDNS check + if: matrix.os == 'macos-latest' + env: + RUST_LOG: nx_core::discovery::mdns=debug,mdns_sd=debug + run: >- + cargo test -p nx-core + discovery::mdns::tests::two_daemons_discover_and_remove_an_announced_endpoint + -- --ignored --exact --nocapture + + - name: Build three-node mDNS guests + if: matrix.os == 'macos-latest' + run: | + rustup target add wasm32-unknown-unknown + cargo build --locked --release --target wasm32-unknown-unknown \ + --manifest-path examples/discovery_lan/Cargo.toml \ + --target-dir examples/discovery_lan/target/reader + cargo build --locked --release --target wasm32-unknown-unknown \ + --manifest-path examples/discovery_lan/Cargo.toml --features increment \ + --target-dir examples/discovery_lan/target/writer + - name: Run discovery demo script tests including real lifecycle + if: matrix.os == 'macos-latest' + env: + NUMAX_DEMO_E2E: "1" + # cargo test built target/debug/nx; both guest variants were built above. + # This requires a real local IPv4 interface, not three separate devices. + run: node --test examples/discovery_lan/demo.test.mjs + + discovery-demo-tests: + name: Discovery Demo Arguments and Configuration + runs-on: macos-latest + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + - name: Check local interface prerequisite + # Initialization validates an actual interface even though these tests + # do not run a daemon or exchange multicast traffic. + run: >- + node --input-type=module -e + "import { networkInterfaces } from 'node:os'; + import assert from 'node:assert/strict'; + assert.ok(Object.values(networkInterfaces()).flat().some(a => a?.family === 'IPv4' && !a.internal), + 'discovery demo configuration tests require a non-loopback local IPv4 interface');" + - name: Test demo arguments and private configuration + run: >- + node --test --test-name-pattern='rejects missing arguments|creates private loopback' + examples/discovery_lan/demo.test.mjs build-wasm: name: Build WASM Examples @@ -140,12 +192,21 @@ jobs: run: cargo build --release --target wasm32-unknown-unknown --manifest-path examples/distributed_counter/Cargo.toml - name: Build previous release binary run: | - git worktree add "${RUNNER_TEMP}/numax-v0.1.0" v0.1.0 - cargo build --release --manifest-path "${RUNNER_TEMP}/numax-v0.1.0/Cargo.toml" -p nx-cli + expected_commit=419d840e2afe780e7ad1f4135e39e9b38a4f30b1 + actual_commit="$(git rev-parse --verify 'refs/tags/v0.1.4^{commit}')" + if [ "$actual_commit" != "$expected_commit" ]; then + echo "Unexpected v0.1.4 tag commit: $actual_commit (expected $expected_commit)" + exit 1 + fi + git worktree add --detach "${RUNNER_TEMP}/numax-v0.1.4" refs/tags/v0.1.4 + cargo build --release --manifest-path "${RUNNER_TEMP}/numax-v0.1.4/Cargo.toml" -p nx-cli - name: Run multi-process CLI smoke test env: - NUMAX_PREVIOUS_NX_BIN: ${{ runner.temp }}/numax-v0.1.0/target/release/nx - run: cargo test -p nx-cli --test multiprocess_smoke -- --ignored + NUMAX_PREVIOUS_NX_BIN: ${{ runner.temp }}/numax-v0.1.4/target/release/nx + # Real multicast is explicitly opted into in the macOS test job above. + run: >- + cargo test -p nx-cli --test multiprocess_smoke + -- --ignored --skip discovery_lan:: benchmark-tools: name: Benchmark Comparator Tests @@ -267,6 +328,7 @@ jobs: - fmt - clippy - test + - discovery-demo-tests - build-wasm - cli-smoke - benchmark-tools @@ -282,6 +344,7 @@ jobs: [[ "${{ needs.fmt.result }}" != "success" ]] || \ [[ "${{ needs.clippy.result }}" != "success" ]] || \ [[ "${{ needs.test.result }}" != "success" ]] || \ + [[ "${{ needs.discovery-demo-tests.result }}" != "success" ]] || \ [[ "${{ needs.build-wasm.result }}" != "success" ]] || \ [[ "${{ needs.cli-smoke.result }}" != "success" ]] || \ [[ "${{ needs.benchmark-tools.result }}" != "success" ]] || \ diff --git a/Cargo.lock b/Cargo.lock index aa8d33f..84a297d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -441,9 +441,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cpufeatures 0.3.0", @@ -523,6 +523,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "combine" +version = "4.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e" +dependencies = [ + "bytes", + "memchr", +] + [[package]] name = "console-api" version = "0.9.0" @@ -575,6 +585,22 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + [[package]] name = "cpp_demangle" version = "0.5.1" @@ -757,6 +783,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.16" @@ -1014,7 +1046,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -1057,6 +1089,17 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flume" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e139bc46ca777eb5efaf62df0ab8cc5fd400866427e56c68b22e414e53bd3be" +dependencies = [ + "futures-core", + "futures-sink", + "spin 0.9.9", +] + [[package]] name = "fnv" version = "1.0.7" @@ -1141,6 +1184,17 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53c0fa8157de1303bfffdaa1cc2a673bfffb60102f76b0ef4441659124373fed" +[[package]] +name = "futures-macro" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "futures-sink" version = "0.3.34" @@ -1162,6 +1216,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-io", + "futures-macro", "futures-sink", "futures-task", "memchr", @@ -1322,6 +1377,76 @@ version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hickory-net" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c480823ed7c2c5d0f09c41020cb6b7c28029ce60ec42dc942158dcf22f8e0a4d" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "futures-channel", + "futures-io", + "futures-util", + "hickory-proto", + "idna", + "ipnet", + "jni", + "rand 0.10.2", + "thiserror 2.0.20", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "hickory-proto" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12b92608f679a6fa515dd1d15c1ff89443026e391200a2c840c7afcba482893d" +dependencies = [ + "data-encoding", + "idna", + "ipnet", + "jni", + "once_cell", + "prefix-trie", + "rand 0.10.2", + "ring", + "thiserror 2.0.20", + "tinyvec", + "tracing", + "url", +] + +[[package]] +name = "hickory-resolver" +version = "0.26.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3da5255c95d5a716857d54b5b8f4e8d67c3484d3beaaaae2ce25063b3ba981" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-net", + "hickory-proto", + "ipconfig", + "ipnet", + "jni", + "moka", + "ndk-context", + "once_cell", + "parking_lot 0.12.5", + "rand 0.10.2", + "resolv-conf", + "smallvec", + "system-configuration", + "thiserror 2.0.20", + "tokio", + "tracing", +] + [[package]] name = "http" version = "1.5.0" @@ -1553,6 +1678,16 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "if-addrs" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0a05c691e1fae256cf7013d99dad472dc52d5543322761f83ec8d47eab40d2b" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + [[package]] name = "indexmap" version = "2.14.0" @@ -1612,11 +1747,27 @@ version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" +[[package]] +name = "ipconfig" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d40460c0ce33d6ce4b0630ad68ff63d6661961c48b6dba35e5a4d81cfb48222" +dependencies = [ + "socket2", + "widestring", + "windows-registry", + "windows-result", + "windows-sys 0.61.2", +] + [[package]] name = "ipnet" version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" +dependencies = [ + "serde", +] [[package]] name = "is-terminal" @@ -1676,6 +1827,55 @@ dependencies = [ "cc", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror 2.0.20", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + [[package]] name = "jobserver" version = "0.1.35" @@ -1790,6 +1990,20 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" +[[package]] +name = "mdns-sd" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a63c9b854b5ad0812ac5969f8db92a59f28b45d0a5ae599b9a45b84c5c8a08e8" +dependencies = [ + "fastrand", + "flume", + "if-addrs", + "mio", + "socket-pktinfo", + "socket2", +] + [[package]] name = "memchr" version = "2.8.3" @@ -1849,10 +2063,34 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" dependencies = [ "libc", + "log", "wasi", "windows-sys 0.61.2", ] +[[package]] +name = "moka" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4293f18e7567a1caf3c584855554377025c65e0aa445344d04171f5ad63d19b9" +dependencies = [ + "crossbeam-channel", + "crossbeam-epoch", + "crossbeam-utils", + "equivalent", + "parking_lot 0.12.5", + "portable-atomic", + "smallvec", + "tagptr", + "uuid", +] + +[[package]] +name = "ndk-context" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" + [[package]] name = "nix" version = "0.26.4" @@ -1938,7 +2176,7 @@ dependencies = [ [[package]] name = "nx-api" -version = "0.1.4" +version = "0.1.5" dependencies = [ "anyhow", "async-trait", @@ -1960,7 +2198,7 @@ dependencies = [ [[package]] name = "nx-cli" -version = "0.1.4" +version = "0.1.5" dependencies = [ "anyhow", "clap", @@ -1979,14 +2217,16 @@ dependencies = [ [[package]] name = "nx-core" -version = "0.1.4" +version = "0.1.5" dependencies = [ "anyhow", "async-trait", "blake3", "dhat", "getrandom 0.4.3", + "hickory-resolver", "inferno", + "mdns-sd", "nx-net", "nx-store", "nx-sync", @@ -2002,7 +2242,7 @@ dependencies = [ [[package]] name = "nx-net" -version = "0.1.4" +version = "0.1.5" dependencies = [ "hex", "nx-sync", @@ -2023,11 +2263,11 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" [[package]] name = "nx-store" -version = "0.1.4" +version = "0.1.5" dependencies = [ "sled", "tempfile", @@ -2036,7 +2276,7 @@ dependencies = [ [[package]] name = "nx-sync" -version = "0.1.4" +version = "0.1.5" dependencies = [ "proptest", "serde", @@ -2081,6 +2321,10 @@ name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "once_cell_polyfill" @@ -2210,6 +2454,12 @@ version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" +[[package]] +name = "portable-atomic" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" + [[package]] name = "postcard" version = "1.1.3" @@ -2252,7 +2502,7 @@ dependencies = [ "nix", "once_cell", "smallvec", - "spin", + "spin 0.10.1", "symbolic-demangle", "tempfile", "thiserror 2.0.20", @@ -2267,6 +2517,17 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prefix-trie" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf6e3177f0684016a5c209b00882e15f8bdd3f3bb48f0491df10cd102d0c6e7" +dependencies = [ + "either", + "ipnet", + "num-traits", +] + [[package]] name = "proc-macro2" version = "1.0.107" @@ -2526,6 +2787,12 @@ version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" +[[package]] +name = "resolv-conf" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" + [[package]] name = "rgb" version = "0.8.53" @@ -2567,6 +2834,15 @@ version = "2.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -2586,7 +2862,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -2601,9 +2877,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.43" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -2647,6 +2923,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -2791,6 +3076,22 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -2822,6 +3123,17 @@ dependencies = [ "serde", ] +[[package]] +name = "socket-pktinfo" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "612942246d0cc239cfd83af1dfd39be47f649208a3524e5e9da651910128e0ac" +dependencies = [ + "libc", + "socket2", + "windows-sys 0.61.2", +] + [[package]] name = "socket2" version = "0.6.5" @@ -2832,6 +3144,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "spin" +version = "0.9.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3763264f6b73151db08c50ff20d7d8a0b8796e021cdea7ceedad07b80155fa0e" +dependencies = [ + "lock_api", +] + [[package]] name = "spin" version = "0.10.1" @@ -2945,6 +3266,33 @@ dependencies = [ "syn 2.0.119", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.13.1", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "tagptr" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b2093cf4c8eb1e67749a6762251bc9cd836b6fc171623bd0a9d324d37af2417" + [[package]] name = "target-lexicon" version = "0.13.5" @@ -2958,10 +3306,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3068,6 +3416,21 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinyvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "tokio" version = "1.53.1" @@ -3434,6 +3797,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" @@ -3885,6 +4258,12 @@ dependencies = [ "wast 256.0.0", ] +[[package]] +name = "widestring" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" + [[package]] name = "wiggle" version = "47.0.4" @@ -3947,7 +4326,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3987,6 +4366,35 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-sys" version = "0.52.0" diff --git a/README.md b/README.md index 83fba26..daecc72 100644 --- a/README.md +++ b/README.md @@ -135,7 +135,7 @@ Same module, any node. State stays local. Sync happens through the runtime. --- -## Learn more +## Learn more and Try Numax ! - [`Documentation`](https://gianiac.github.io/numax/) - guides, concepts and reference pages. - [`Whitepaper`](https://gianiac.github.io/numax/whitepaper/) - the vision, the architecture, the principles. @@ -143,6 +143,7 @@ Same module, any node. State stays local. Sync happens through the runtime. - [`Host API`](https://gianiac.github.io/numax/reference/host-api/) - the host API available to WASM modules. - [`examples/distributed_magnets`](./examples/distributed_magnets) - adaptive Magnetic Optimization Algorithm swarm. - [`examples/distributed_ants`](./examples/distributed_ants) - distributed Ant Colony Optimization swarm. +- [`examples/discovery_lan`](./examples/discovery_lan) - LAN discovery, CRDT replication and restart recovery - [`examples/distributed_inventory`](./examples/distributed_inventory) - replicated PNCounter inventory. - [`examples/distributed_status`](./examples/distributed_status) - replicated LWW-Register status. - [`examples/distributed_tags`](./examples/distributed_tags) - replicated ORSet tags. diff --git a/crates/nx-api/Cargo.toml b/crates/nx-api/Cargo.toml index 4bcffe9..9e1646c 100644 --- a/crates/nx-api/Cargo.toml +++ b/crates/nx-api/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nx-api" -version = "0.1.4" +version = "0.1.5" edition = "2024" license.workspace = true @@ -11,7 +11,7 @@ base64 = "0.22" hyper = { version = "1", features = ["http1", "server"] } hyper-util = { version = "0.1", features = ["tokio"] } http-body-util = "0.1" -nx-core = { version = "0.1.4", path = "../nx-core" } +nx-core = { version = "0.1.5", path = "../nx-core" } serde = { version = "1", features = ["derive"] } serde_json = "1" subtle = "2" diff --git a/crates/nx-cli/Cargo.toml b/crates/nx-cli/Cargo.toml index 4774278..654ca08 100644 --- a/crates/nx-cli/Cargo.toml +++ b/crates/nx-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nx-cli" -version = "0.1.4" +version = "0.1.5" edition = "2024" license.workspace = true @@ -13,8 +13,8 @@ anyhow = "1" clap = { version = "4", features = ["derive"] } clap_complete = "4" console-subscriber = { version = "0.5", optional = true } -nx-core = { version = "0.1.4", path = "../nx-core" } -nx-api = { version = "0.1.4", path = "../nx-api" } +nx-core = { version = "0.1.5", path = "../nx-core" } +nx-api = { version = "0.1.5", path = "../nx-api" } owo-colors = { version = "4", features = ["supports-colors"] } serde = { version = "1", features = ["derive"] } tokio = { version = "1", features = ["rt-multi-thread", "macros"] } diff --git a/crates/nx-cli/src/config.rs b/crates/nx-cli/src/config.rs index 66a2386..7a02412 100644 --- a/crates/nx-cli/src/config.rs +++ b/crates/nx-cli/src/config.rs @@ -7,7 +7,12 @@ use anyhow::{Context, Result, bail}; use clap::ValueEnum; use nx_api::{DEFAULT_MANAGEMENT_LISTEN, DEFAULT_MANAGEMENT_REQUEST_TIMEOUT, ManagementConfig}; use nx_core::runtime::RuntimeConfig; -use nx_core::{ObservabilityConfig, SerializationFormat, SyncConfig, TlsConfig}; +use nx_core::{ + BootstrapDiscoverySettings, DnsSrvDiscoverySettings, FileDiscoverySettings, + MAX_BOOTSTRAP_RESPONSE_CAPACITY as MAX_BOOTSTRAP_CANDIDATES, MdnsDiscoverySettings, + ObservabilityConfig, RuntimeDiscoveryConfig, RuntimeDiscoveryMode, SerializationFormat, + SyncConfig, TlsConfig, +}; use serde::Deserialize; use tracing::warn; #[cfg(feature = "tokio-console")] @@ -100,12 +105,33 @@ pub(crate) struct ManagementFileConfig { #[serde(deny_unknown_fields)] pub(crate) struct DiscoveryFileConfig { pub(crate) mode: Option, + pub(crate) cluster_id: Option, + pub(crate) advertised_endpoint: Option, + pub(crate) max_candidates: Option, + pub(crate) seeds: Option>, + pub(crate) refresh_interval: Option, + pub(crate) retry_initial: Option, + pub(crate) retry_max: Option, + pub(crate) stale_after: Option, + pub(crate) max_seeds: Option, + pub(crate) instance_name: Option, + pub(crate) max_instances: Option, + pub(crate) service_name: Option, + pub(crate) retry_interval: Option, + pub(crate) max_refresh_interval: Option, + pub(crate) path: Option, + pub(crate) poll_interval: Option, + pub(crate) max_file_bytes: Option, } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, ValueEnum)] #[serde(rename_all = "kebab-case")] pub(crate) enum DiscoveryMode { Static, + Bootstrap, + Mdns, + DnsSrv, + File, } #[derive(Debug, Default)] @@ -123,6 +149,11 @@ pub(crate) struct RunCliOptions { pub(crate) verbose: bool, pub(crate) log_level: Option, pub(crate) log_format: Option, + pub(crate) discovery_mode: Option, + pub(crate) bootstrap_seeds: Vec, + pub(crate) mdns_instance: Option, + pub(crate) dns_srv_name: Option, + pub(crate) peer_file: Option, } #[derive(Debug)] @@ -131,6 +162,7 @@ pub(crate) struct EffectiveRunConfig { pub(crate) sync: Option, pub(crate) observability: Option, pub(crate) management: Option, + pub(crate) discovery: RuntimeDiscoveryConfig, pub(crate) log_level: String, pub(crate) log_format: LogFormat, } @@ -147,6 +179,8 @@ impl EffectiveRunConfig { file_config: &RunFileConfig, ) -> Result { let env_has_sync_inputs = env_config.has_sync_inputs(); + let mut discovery = + resolve_discovery_config(&cli, &env_config, file_config.discovery.as_ref())?; let management = build_management_config(&env_config, file_config.management.as_ref())?; let datastore_path = cli .datastore_path @@ -212,6 +246,11 @@ impl EffectiveRunConfig { .and_then(|network| network.peers.clone()) .unwrap_or_default() }; + discovery.max_candidates = discovery.max_candidates.max(peers.len()); + validate_discovery_candidate_capacity( + discovery.max_candidates, + matches!(discovery.mode, RuntimeDiscoveryMode::Bootstrap(_)), + )?; let serialization_format = if cli.debug_protocol { Some(SerializationFormat::Json) } else if let Some(format) = env_config.serialization_format { @@ -228,6 +267,7 @@ impl EffectiveRunConfig { || file_config.tls.is_some() || file_config.network.is_some() || env_has_sync_inputs + || !matches!(discovery.mode, RuntimeDiscoveryMode::Static) || tls.is_some() || serialization_format.is_some(); let sync = build_sync_config(listen, peers, tls, force_sync, serialization_format)? @@ -243,6 +283,7 @@ impl EffectiveRunConfig { sync, observability, management, + discovery, log_level, log_format, }) @@ -389,7 +430,7 @@ impl EffectiveRunConfig { } out.push_str("[discovery]\n"); - out.push_str("mode = \"static\"\n"); + render_discovery_config(&mut out, &self.discovery); out } } @@ -438,6 +479,13 @@ anti_entropy_interval = "30s" [discovery] mode = "static" +# cluster_id = "default" +# advertised_endpoint = "127.0.0.1:9000" +# max_candidates = 1024 +# Bootstrap: seeds, refresh_interval, retry_initial, retry_max, stale_after, max_seeds +# mDNS: instance_name, max_instances +# DNS-SRV: service_name, retry_interval, max_refresh_interval +# File: path, poll_interval, max_file_bytes "#; pub(crate) fn init_config_file(path: &Path, force: bool) -> Result<()> { @@ -471,6 +519,24 @@ pub(crate) struct EnvRunConfig { pub(crate) serialization_format: Option, pub(crate) log_level: Option, pub(crate) log_format: Option, + pub(crate) discovery_mode: Option, + pub(crate) discovery_cluster_id: Option, + pub(crate) discovery_advertised_endpoint: Option, + pub(crate) discovery_max_candidates: Option, + pub(crate) discovery_seeds: Option>, + pub(crate) discovery_refresh_interval: Option, + pub(crate) discovery_retry_initial: Option, + pub(crate) discovery_retry_max: Option, + pub(crate) discovery_stale_after: Option, + pub(crate) discovery_max_seeds: Option, + pub(crate) discovery_instance_name: Option, + pub(crate) discovery_max_instances: Option, + pub(crate) discovery_service_name: Option, + pub(crate) discovery_retry_interval: Option, + pub(crate) discovery_max_refresh_interval: Option, + pub(crate) discovery_file: Option, + pub(crate) discovery_poll_interval: Option, + pub(crate) discovery_max_file_bytes: Option, } impl EnvRunConfig { @@ -493,6 +559,24 @@ impl EnvRunConfig { serialization_format: env_serialization_format()?, log_level: env_non_empty("NX_LOG_LEVEL")?, log_format: env_log_format()?, + discovery_mode: env_discovery_mode()?, + discovery_cluster_id: env_non_empty("NX_DISCOVERY_CLUSTER_ID")?, + discovery_advertised_endpoint: env_non_empty("NX_DISCOVERY_ADVERTISED_ENDPOINT")?, + discovery_max_candidates: env_usize("NX_DISCOVERY_MAX_CANDIDATES")?, + discovery_seeds: env_csv("NX_DISCOVERY_SEEDS")?, + discovery_refresh_interval: env_non_empty("NX_DISCOVERY_REFRESH_INTERVAL")?, + discovery_retry_initial: env_non_empty("NX_DISCOVERY_RETRY_INITIAL")?, + discovery_retry_max: env_non_empty("NX_DISCOVERY_RETRY_MAX")?, + discovery_stale_after: env_non_empty("NX_DISCOVERY_STALE_AFTER")?, + discovery_max_seeds: env_usize("NX_DISCOVERY_MAX_SEEDS")?, + discovery_instance_name: env_non_empty("NX_DISCOVERY_INSTANCE_NAME")?, + discovery_max_instances: env_usize("NX_DISCOVERY_MAX_INSTANCES")?, + discovery_service_name: env_non_empty("NX_DISCOVERY_SERVICE_NAME")?, + discovery_retry_interval: env_non_empty("NX_DISCOVERY_RETRY_INTERVAL")?, + discovery_max_refresh_interval: env_non_empty("NX_DISCOVERY_MAX_REFRESH_INTERVAL")?, + discovery_file: env_path("NX_DISCOVERY_FILE"), + discovery_poll_interval: env_non_empty("NX_DISCOVERY_POLL_INTERVAL")?, + discovery_max_file_bytes: env_non_empty("NX_DISCOVERY_MAX_FILE_BYTES")?, }) } @@ -505,6 +589,14 @@ impl EnvRunConfig { || self.allowed_peers.is_some() || self.tls_insecure.unwrap_or(false) || self.serialization_format.is_some() + || self.discovery_mode.is_some() + || self.discovery_cluster_id.is_some() + || self.discovery_advertised_endpoint.is_some() + || self.discovery_max_candidates.is_some() + || self.discovery_seeds.is_some() + || self.discovery_instance_name.is_some() + || self.discovery_service_name.is_some() + || self.discovery_file.is_some() } } @@ -571,6 +663,85 @@ fn render_log_format(format: LogFormat) -> &'static str { } } +fn render_discovery_config(out: &mut String, config: &RuntimeDiscoveryConfig) { + let mode = match &config.mode { + RuntimeDiscoveryMode::Static => "static", + RuntimeDiscoveryMode::Bootstrap(_) => "bootstrap", + RuntimeDiscoveryMode::Mdns(_) => "mdns", + RuntimeDiscoveryMode::DnsSrv(_) => "dns-srv", + RuntimeDiscoveryMode::File(_) => "file", + }; + out.push_str(&format!("mode = \"{mode}\"\n")); + out.push_str(&format!( + "cluster_id = \"{}\"\n", + escape_toml(&config.cluster_id) + )); + render_optional_string( + out, + "advertised_endpoint", + config.advertised_endpoint.as_deref(), + ); + out.push_str(&format!("max_candidates = {}\n", config.max_candidates)); + match &config.mode { + RuntimeDiscoveryMode::Static => {} + RuntimeDiscoveryMode::Bootstrap(settings) => { + out.push_str(&format!( + "seeds = {}\n", + render_string_list(&settings.seeds) + )); + out.push_str(&format!( + "refresh_interval = \"{}\"\n", + render_duration(settings.refresh_interval) + )); + out.push_str(&format!( + "retry_initial = \"{}\"\n", + render_duration(settings.retry_initial) + )); + out.push_str(&format!( + "retry_max = \"{}\"\n", + render_duration(settings.retry_max) + )); + out.push_str(&format!( + "stale_after = \"{}\"\n", + render_duration(settings.stale_after) + )); + out.push_str(&format!("max_seeds = {}\n", settings.max_seeds)); + } + RuntimeDiscoveryMode::Mdns(settings) => { + out.push_str(&format!( + "instance_name = \"{}\"\n", + escape_toml(&settings.instance_name) + )); + out.push_str(&format!("max_instances = {}\n", settings.max_instances)); + } + RuntimeDiscoveryMode::DnsSrv(settings) => { + out.push_str(&format!( + "service_name = \"{}\"\n", + escape_toml(&settings.service_name) + )); + out.push_str(&format!( + "retry_interval = \"{}\"\n", + render_duration(settings.retry_interval) + )); + out.push_str(&format!( + "max_refresh_interval = \"{}\"\n", + render_duration(settings.max_refresh_interval) + )); + } + RuntimeDiscoveryMode::File(settings) => { + out.push_str(&format!( + "path = \"{}\"\n", + escape_toml(&settings.path.to_string_lossy()) + )); + out.push_str(&format!( + "poll_interval = \"{}\"\n", + render_duration(settings.poll_interval) + )); + out.push_str(&format!("max_file_bytes = {}\n", settings.max_file_bytes)); + } + } +} + fn render_duration(duration: Duration) -> String { let millis = duration.as_millis(); if millis.is_multiple_of(60_000) { @@ -622,6 +793,304 @@ fn env_peers() -> Result>> { } } +fn env_csv(name: &str) -> Result>> { + let Some(value) = env_non_empty(name)? else { + return Ok(None); + }; + value + .split(',') + .map(|item| { + validate_non_empty(name, item)?; + Ok(item.trim().to_string()) + }) + .collect::>>() + .map(Some) +} + +fn env_usize(name: &str) -> Result> { + let Some(value) = env_non_empty(name)? else { + return Ok(None); + }; + value + .parse::() + .map(Some) + .with_context(|| format!("{name} must be an unsigned integer")) +} + +fn env_discovery_mode() -> Result> { + let Some(value) = env_non_empty("NX_DISCOVERY_MODE")? else { + return Ok(None); + }; + match value.to_ascii_lowercase().as_str() { + "static" => Ok(Some(DiscoveryMode::Static)), + "bootstrap" => Ok(Some(DiscoveryMode::Bootstrap)), + "mdns" => Ok(Some(DiscoveryMode::Mdns)), + "dns-srv" => Ok(Some(DiscoveryMode::DnsSrv)), + "file" => Ok(Some(DiscoveryMode::File)), + _ => bail!("NX_DISCOVERY_MODE must be one of static, bootstrap, mdns, dns-srv, file"), + } +} + +fn resolve_discovery_config( + cli: &RunCliOptions, + env: &EnvRunConfig, + file: Option<&DiscoveryFileConfig>, +) -> Result { + let mode = cli + .discovery_mode + .or(env.discovery_mode) + .or_else(|| file.and_then(|config| config.mode)) + .unwrap_or(DiscoveryMode::Static); + validate_discovery_mode_fields(mode, cli, env, file)?; + + let cluster_id = env + .discovery_cluster_id + .clone() + .or_else(|| file.and_then(|config| config.cluster_id.clone())) + .unwrap_or_else(|| nx_core::DEFAULT_DISCOVERY_CLUSTER.to_string()); + let advertised_endpoint = env + .discovery_advertised_endpoint + .clone() + .or_else(|| file.and_then(|config| config.advertised_endpoint.clone())); + let max_candidates = env + .discovery_max_candidates + .or_else(|| file.and_then(|config| config.max_candidates)) + .unwrap_or(nx_core::DEFAULT_MAX_PEER_CANDIDATES); + validate_non_empty("discovery.cluster_id", &cluster_id)?; + validate_optional_non_empty( + "discovery.advertised_endpoint", + advertised_endpoint.as_deref(), + )?; + validate_discovery_candidate_capacity(max_candidates, mode == DiscoveryMode::Bootstrap)?; + + let resolved_mode = match mode { + DiscoveryMode::Static => RuntimeDiscoveryMode::Static, + DiscoveryMode::Bootstrap => { + let seeds = if !cli.bootstrap_seeds.is_empty() { + cli.bootstrap_seeds.clone() + } else { + env.discovery_seeds + .clone() + .or_else(|| file.and_then(|config| config.seeds.clone())) + .unwrap_or_default() + }; + if seeds.is_empty() { + bail!("discovery.seeds is required when discovery.mode = \"bootstrap\""); + } + for seed in &seeds { + validate_non_empty("discovery.seeds", seed)?; + } + let mut settings = BootstrapDiscoverySettings::new(seeds); + settings.refresh_interval = resolve_discovery_duration( + env.discovery_refresh_interval.as_deref(), + file.and_then(|config| config.refresh_interval.as_deref()), + settings.refresh_interval, + "discovery.refresh_interval", + )?; + settings.retry_initial = resolve_discovery_duration( + env.discovery_retry_initial.as_deref(), + file.and_then(|config| config.retry_initial.as_deref()), + settings.retry_initial, + "discovery.retry_initial", + )?; + settings.retry_max = resolve_discovery_duration( + env.discovery_retry_max.as_deref(), + file.and_then(|config| config.retry_max.as_deref()), + settings.retry_max, + "discovery.retry_max", + )?; + settings.stale_after = resolve_discovery_duration( + env.discovery_stale_after.as_deref(), + file.and_then(|config| config.stale_after.as_deref()), + settings.stale_after, + "discovery.stale_after", + )?; + settings.max_seeds = env + .discovery_max_seeds + .or_else(|| file.and_then(|config| config.max_seeds)) + .unwrap_or(settings.max_seeds); + validate_non_zero("discovery.max_seeds", settings.max_seeds)?; + if settings.retry_initial > settings.retry_max { + bail!("discovery.retry_initial must be less than or equal to discovery.retry_max"); + } + RuntimeDiscoveryMode::Bootstrap(settings) + } + DiscoveryMode::Mdns => { + let instance_name = cli + .mdns_instance + .clone() + .or_else(|| env.discovery_instance_name.clone()) + .or_else(|| file.and_then(|config| config.instance_name.clone())) + .context("discovery.instance_name is required when discovery.mode = \"mdns\"")?; + validate_non_empty("discovery.instance_name", &instance_name)?; + let mut settings = MdnsDiscoverySettings::new(instance_name); + settings.max_instances = env + .discovery_max_instances + .or_else(|| file.and_then(|config| config.max_instances)) + .unwrap_or(settings.max_instances); + validate_non_zero("discovery.max_instances", settings.max_instances)?; + RuntimeDiscoveryMode::Mdns(settings) + } + DiscoveryMode::DnsSrv => { + let service_name = cli + .dns_srv_name + .clone() + .or_else(|| env.discovery_service_name.clone()) + .or_else(|| file.and_then(|config| config.service_name.clone())) + .context("discovery.service_name is required when discovery.mode = \"dns-srv\"")?; + validate_non_empty("discovery.service_name", &service_name)?; + let mut settings = DnsSrvDiscoverySettings::new(service_name); + settings.retry_interval = resolve_discovery_duration( + env.discovery_retry_interval.as_deref(), + file.and_then(|config| config.retry_interval.as_deref()), + settings.retry_interval, + "discovery.retry_interval", + )?; + settings.max_refresh_interval = resolve_discovery_duration( + env.discovery_max_refresh_interval.as_deref(), + file.and_then(|config| config.max_refresh_interval.as_deref()), + settings.max_refresh_interval, + "discovery.max_refresh_interval", + )?; + RuntimeDiscoveryMode::DnsSrv(settings) + } + DiscoveryMode::File => { + let path = cli + .peer_file + .clone() + .or_else(|| env.discovery_file.clone()) + .or_else(|| file.and_then(|config| config.path.clone())) + .context("discovery.path is required when discovery.mode = \"file\"")?; + validate_optional_path("discovery.path", Some(&path))?; + let mut settings = FileDiscoverySettings::new(path); + settings.poll_interval = resolve_discovery_duration( + env.discovery_poll_interval.as_deref(), + file.and_then(|config| config.poll_interval.as_deref()), + settings.poll_interval, + "discovery.poll_interval", + )?; + settings.max_file_bytes = env + .discovery_max_file_bytes + .as_deref() + .or_else(|| file.and_then(|config| config.max_file_bytes.as_deref())) + .map(parse_byte_size) + .transpose()? + .unwrap_or(settings.max_file_bytes); + RuntimeDiscoveryMode::File(settings) + } + }; + + Ok(RuntimeDiscoveryConfig { + cluster_id, + advertised_endpoint, + max_candidates, + mode: resolved_mode, + }) +} + +fn validate_discovery_candidate_capacity(max_candidates: usize, bootstrap: bool) -> Result<()> { + if max_candidates == 0 { + bail!("discovery.max_candidates must be greater than zero"); + } + if bootstrap && max_candidates > MAX_BOOTSTRAP_CANDIDATES { + bail!( + "discovery.max_candidates must be at most {MAX_BOOTSTRAP_CANDIDATES} when discovery.mode = \"bootstrap\"" + ); + } + Ok(()) +} + +fn resolve_discovery_duration( + env: Option<&str>, + file: Option<&str>, + default: Duration, + name: &str, +) -> Result { + env.or(file) + .map(|value| parse_duration(value).map_err(|error| anyhow::anyhow!("{name}: {error}"))) + .transpose() + .map(|duration| duration.unwrap_or(default)) +} + +fn validate_discovery_mode_fields( + mode: DiscoveryMode, + cli: &RunCliOptions, + env: &EnvRunConfig, + file: Option<&DiscoveryFileConfig>, +) -> Result<()> { + let include_env = cli.discovery_mode.is_none(); + let include_file = include_env && env.discovery_mode.is_none(); + let bootstrap = !cli.bootstrap_seeds.is_empty() + || include_env + && (env.discovery_seeds.is_some() + || env.discovery_refresh_interval.is_some() + || env.discovery_retry_initial.is_some() + || env.discovery_retry_max.is_some() + || env.discovery_stale_after.is_some() + || env.discovery_max_seeds.is_some()) + || include_file + && file.is_some_and(|config| { + config.seeds.is_some() + || config.refresh_interval.is_some() + || config.retry_initial.is_some() + || config.retry_max.is_some() + || config.stale_after.is_some() + || config.max_seeds.is_some() + }); + let mdns = cli.mdns_instance.is_some() + || include_env + && (env.discovery_instance_name.is_some() || env.discovery_max_instances.is_some()) + || include_file + && file.is_some_and(|config| { + config.instance_name.is_some() || config.max_instances.is_some() + }); + let dns_srv = cli.dns_srv_name.is_some() + || include_env + && (env.discovery_service_name.is_some() + || env.discovery_retry_interval.is_some() + || env.discovery_max_refresh_interval.is_some()) + || include_file + && file.is_some_and(|config| { + config.service_name.is_some() + || config.retry_interval.is_some() + || config.max_refresh_interval.is_some() + }); + let file_watch = cli.peer_file.is_some() + || include_env + && (env.discovery_file.is_some() + || env.discovery_poll_interval.is_some() + || env.discovery_max_file_bytes.is_some()) + || include_file + && file.is_some_and(|config| { + config.path.is_some() + || config.poll_interval.is_some() + || config.max_file_bytes.is_some() + }); + let invalid = match mode { + DiscoveryMode::Static => bootstrap || mdns || dns_srv || file_watch, + DiscoveryMode::Bootstrap => mdns || dns_srv || file_watch, + DiscoveryMode::Mdns => bootstrap || dns_srv || file_watch, + DiscoveryMode::DnsSrv => bootstrap || mdns || file_watch, + DiscoveryMode::File => bootstrap || mdns || dns_srv, + }; + if invalid { + bail!("discovery contains fields that are not valid for mode {mode}"); + } + Ok(()) +} + +impl std::fmt::Display for DiscoveryMode { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(match self { + Self::Static => "static", + Self::Bootstrap => "bootstrap", + Self::Mdns => "mdns", + Self::DnsSrv => "dns-srv", + Self::File => "file", + }) + } +} + fn env_bool(name: &str) -> Result> { let Some(value) = env_non_empty(name)? else { return Ok(None); @@ -803,11 +1272,11 @@ pub(crate) fn validate_run_file_config(config: &RunFileConfig) -> Result<()> { )?; } - if let Some(discovery) = &config.discovery { - match discovery.mode { - Some(DiscoveryMode::Static) | None => {} - } - } + resolve_discovery_config( + &RunCliOptions::default(), + &EnvRunConfig::default(), + config.discovery.as_ref(), + )?; Ok(()) } @@ -819,6 +1288,13 @@ fn validate_non_empty(name: &str, value: &str) -> Result<()> { Ok(()) } +fn validate_non_zero(name: &str, value: usize) -> Result<()> { + if value == 0 { + bail!("{name} must be greater than zero"); + } + Ok(()) +} + fn validate_optional_non_empty(name: &str, value: Option<&str>) -> Result<()> { if let Some(value) = value { validate_non_empty(name, value)?; @@ -1311,3 +1787,132 @@ pub(crate) fn build_sync_config( debug_assert!(cfg.is_enabled()); Ok(Some(cfg)) } + +#[cfg(test)] +mod tests { + use super::*; + + fn discovery_file(mode: &str, max_candidates: usize) -> RunFileConfig { + let fields = match mode { + "bootstrap" => "seeds = [\"127.0.0.1:9001\"]", + "mdns" => "instance_name = \"config-test\"", + "dns-srv" => "service_name = \"_numax._tcp.example.org.\"", + "file" => "path = \"peers.txt\"", + _ => "", + }; + toml::from_str(&format!( + "[network]\nlisten = \"127.0.0.1:9000\"\n[discovery]\nmode = \"{mode}\"\nmax_candidates = {max_candidates}\n{fields}" + )) + .unwrap() + } + + #[test] + fn bootstrap_candidate_capacity_accepts_boundaries_and_rejects_overflow() { + for capacity in [1, 4_096] { + let effective = EffectiveRunConfig::resolve_with_env( + RunCliOptions::default(), + EnvRunConfig::default(), + &discovery_file("bootstrap", capacity), + ) + .unwrap(); + assert_eq!(effective.discovery.max_candidates, capacity); + } + for capacity in [4_097, usize::MAX] { + let error = EffectiveRunConfig::resolve_with_env( + RunCliOptions::default(), + EnvRunConfig { + discovery_max_candidates: Some(capacity), + ..Default::default() + }, + &discovery_file("bootstrap", 1), + ) + .unwrap_err(); + assert!(error.to_string().contains("at most 4096")); + } + assert!( + EffectiveRunConfig::resolve_with_env( + RunCliOptions::default(), + EnvRunConfig::default(), + &discovery_file("bootstrap", 4_097), + ) + .is_err() + ); + } + + #[test] + fn candidate_capacity_rejects_zero_for_every_mode() { + for mode in ["static", "bootstrap", "mdns", "dns-srv", "file"] { + let error = EffectiveRunConfig::resolve_with_env( + RunCliOptions::default(), + EnvRunConfig::default(), + &discovery_file(mode, 0), + ) + .unwrap_err(); + assert!(error.to_string().contains("greater than zero"), "{mode}"); + } + } + + #[test] + fn candidate_capacity_above_bootstrap_bound_is_valid_for_other_modes() { + for mode in ["static", "mdns", "dns-srv", "file"] { + let effective = EffectiveRunConfig::resolve_with_env( + RunCliOptions::default(), + EnvRunConfig::default(), + &discovery_file(mode, 4_097), + ) + .unwrap(); + assert_eq!(effective.discovery.max_candidates, 4_097, "{mode}"); + } + } + + #[test] + fn bootstrap_capacity_uses_effective_precedence() { + let file = discovery_file("bootstrap", 4_097); + let effective = EffectiveRunConfig::resolve_with_env( + RunCliOptions::default(), + EnvRunConfig { + discovery_max_candidates: Some(4_096), + ..Default::default() + }, + &file, + ) + .unwrap(); + assert_eq!(effective.discovery.max_candidates, 4_096); + + let effective = EffectiveRunConfig::resolve_with_env( + RunCliOptions { + discovery_mode: Some(DiscoveryMode::Static), + ..Default::default() + }, + EnvRunConfig::default(), + &file, + ) + .unwrap(); + assert_eq!(effective.discovery.max_candidates, 4_097); + assert!(matches!( + effective.discovery.mode, + RuntimeDiscoveryMode::Static + )); + } + + #[test] + fn explicit_peers_cannot_expand_bootstrap_capacity_past_wire_bound() { + for mode in ["bootstrap", "static"] { + let result = EffectiveRunConfig::resolve_with_env( + RunCliOptions { + peers: (1..=4_097) + .map(|port| format!("127.0.0.1:{port}")) + .collect(), + ..Default::default() + }, + EnvRunConfig::default(), + &discovery_file(mode, 1), + ); + if mode == "bootstrap" { + assert!(result.unwrap_err().to_string().contains("at most 4096")); + } else { + assert_eq!(result.unwrap().discovery.max_candidates, 4_097); + } + } + } +} diff --git a/crates/nx-cli/src/main.rs b/crates/nx-cli/src/main.rs index 131365b..06a132c 100644 --- a/crates/nx-cli/src/main.rs +++ b/crates/nx-cli/src/main.rs @@ -43,6 +43,26 @@ struct NodeArgs { #[arg(long = "peer", value_name = "ADDR")] peers: Vec, + /// Peer discovery provider. + #[arg(long, value_enum, value_name = "MODE")] + discovery_mode: Option, + + /// Bootstrap endpoint (can be repeated). + #[arg(long = "bootstrap-seed", value_name = "URL")] + bootstrap_seeds: Vec, + + /// mDNS instance name advertised by this node. + #[arg(long, value_name = "NAME")] + mdns_instance: Option, + + /// DNS-SRV service name to resolve. + #[arg(long, value_name = "NAME")] + dns_srv_name: Option, + + /// Path to the watched peer list. + #[arg(long, value_name = "PATH")] + peer_file: Option, + /// Maximum time allowed for shutdown before returning an error. #[arg(long, value_name = "DURATION", value_parser = parse_duration)] shutdown_timeout: Option, @@ -367,6 +387,11 @@ fn resolve_node_args(node: NodeArgs) -> Result { config, listen, peers, + discovery_mode, + bootstrap_seeds, + mdns_instance, + dns_srv_name, + peer_file, shutdown_timeout, verbose, log_level, @@ -395,6 +420,11 @@ fn resolve_node_args(node: NodeArgs) -> Result { verbose, log_level, log_format, + discovery_mode, + bootstrap_seeds, + mdns_instance, + dns_srv_name, + peer_file, }; Ok(ResolvedNodeArgs { @@ -407,7 +437,8 @@ fn resolve_node_args(node: NodeArgs) -> Result { fn runtime_config_from_effective( effective: EffectiveRunConfig, module_id: Option, -) -> RuntimeConfig { +) -> (RuntimeConfig, nx_core::RuntimeDiscoveryConfig) { + let discovery = effective.discovery; let mut config = RuntimeConfig::default(); if let Some(path) = effective.datastore_path { config.datastore_path = path; @@ -426,7 +457,7 @@ fn runtime_config_from_effective( config.sync = Some(sync); } config.observability = effective.observability; - config + (config, discovery) } async fn real_main(cli: Cli) -> Result<()> { @@ -465,12 +496,11 @@ async fn real_main(cli: Cli) -> Result<()> { // Read the wasm module let bytes = fs::read(&module)?; - let cfg = runtime_config_from_effective( + let (cfg, discovery) = runtime_config_from_effective( effective, Some(module.to_string_lossy().into_owned()), ); - - let mut rt = Runtime::new(cfg)?; + let mut rt = Runtime::new_with_discovery(cfg, discovery)?; let run_result: Result<()> = async { rt.start_observability().await?; rt.start_sync().await?; @@ -560,8 +590,8 @@ async fn real_main(cli: Cli) -> Result<()> { let has_active_service = effective.sync.is_some() || effective.observability.is_some() || management_config.is_some(); - let cfg = runtime_config_from_effective(effective, None); - let mut rt = Runtime::new(cfg)?; + let (cfg, discovery) = runtime_config_from_effective(effective, None); + let mut rt = Runtime::new_with_discovery(cfg, discovery)?; let mut management_server = None; let serve_result: Result<()> = async { rt.start_observability().await?; @@ -781,6 +811,11 @@ mod tests { verbose: false, log_level: None, log_format: None, + discovery_mode: None, + bootstrap_seeds: Vec::new(), + mdns_instance: None, + dns_srv_name: None, + peer_file: None, } } @@ -989,6 +1024,142 @@ mod tests { assert_eq!(sync.peers, vec!["cli:1".to_string()]); } + #[test] + fn discovery_precedence_is_cli_then_env_then_file() { + let file_config: RunFileConfig = toml::from_str( + r#" + [network] + listen = "127.0.0.1:9000" + + [discovery] + mode = "mdns" + instance_name = "from-file" + "#, + ) + .unwrap(); + let mut cli = cli_defaults(); + cli.mdns_instance = Some("from-cli".into()); + let env_config = EnvRunConfig { + discovery_instance_name: Some("from-env".into()), + ..EnvRunConfig::default() + }; + + let effective = + EffectiveRunConfig::resolve_with_env(cli, env_config, &file_config).unwrap(); + + let nx_core::RuntimeDiscoveryMode::Mdns(settings) = effective.discovery.mode else { + panic!("expected mdns discovery"); + }; + assert_eq!(settings.instance_name, "from-cli"); + } + + #[test] + fn cli_discovery_mode_ignores_lower_priority_provider_fields() { + let file_config: RunFileConfig = toml::from_str( + r#" + [network] + listen = "127.0.0.1:9000" + + [discovery] + mode = "mdns" + instance_name = "from-file" + "#, + ) + .unwrap(); + let mut cli = cli_defaults(); + cli.discovery_mode = Some(DiscoveryMode::Static); + + let effective = + EffectiveRunConfig::resolve_with_env(cli, EnvRunConfig::default(), &file_config) + .unwrap(); + + assert!(matches!( + effective.discovery.mode, + nx_core::RuntimeDiscoveryMode::Static + )); + } + + #[test] + fn discovery_rejects_fields_from_another_mode() { + let file_config: RunFileConfig = toml::from_str( + r#" + [discovery] + mode = "static" + service_name = "_numax._tcp.example.com" + "#, + ) + .unwrap(); + + let error = EffectiveRunConfig::resolve_with_env( + cli_defaults(), + EnvRunConfig::default(), + &file_config, + ) + .unwrap_err(); + + assert!(error.to_string().contains("not valid for mode static")); + } + + #[test] + fn dynamic_discovery_requires_provider_selector() { + let file_config: RunFileConfig = toml::from_str( + r#" + [network] + listen = "127.0.0.1:9000" + + [discovery] + mode = "dns-srv" + "#, + ) + .unwrap(); + + let error = EffectiveRunConfig::resolve_with_env( + cli_defaults(), + EnvRunConfig::default(), + &file_config, + ) + .unwrap_err(); + + assert!( + error + .to_string() + .contains("discovery.service_name is required") + ); + } + + #[test] + fn dynamic_discovery_keeps_explicit_peers_and_renders_effective_values() { + let file_config: RunFileConfig = toml::from_str( + r#" + [network] + listen = "127.0.0.1:9000" + peers = ["127.0.0.1:9001"] + + [discovery] + mode = "bootstrap" + seeds = ["127.0.0.1:9100"] + refresh_interval = "15s" + "#, + ) + .unwrap(); + + let effective = EffectiveRunConfig::resolve_with_env( + cli_defaults(), + EnvRunConfig::default(), + &file_config, + ) + .unwrap(); + + assert_eq!( + effective.sync.as_ref().unwrap().peers, + vec!["127.0.0.1:9001"] + ); + let rendered = effective.render_effective_toml(); + assert!(rendered.contains("mode = \"bootstrap\"")); + assert!(rendered.contains("seeds = [\"127.0.0.1:9100\"]")); + assert!(rendered.contains("refresh_interval = \"15s\"")); + } + #[test] fn parses_observability_toml() { let cfg: RunFileConfig = toml::from_str( @@ -1068,6 +1239,7 @@ mod tests { sync: None, observability: None, management: Some(management), + discovery: nx_core::RuntimeDiscoveryConfig::default(), log_level: "info".into(), log_format: LogFormat::Text, }; diff --git a/crates/nx-cli/tests/multiprocess_smoke.rs b/crates/nx-cli/tests/multiprocess_smoke.rs index 40db737..4ebc7c8 100644 --- a/crates/nx-cli/tests/multiprocess_smoke.rs +++ b/crates/nx-cli/tests/multiprocess_smoke.rs @@ -9,6 +9,10 @@ use std::time::{SystemTime, UNIX_EPOCH}; const COUNTER_KEY: &str = "counter:visits"; +#[cfg(unix)] +#[path = "support/discovery_lan.rs"] +mod discovery_lan; + fn workspace_root() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")) .parent() @@ -679,14 +683,14 @@ fn two_nx_run_processes_converge_distributed_counter() { } #[test] -#[ignore = "requires v0.1.0 nx binary, built distributed_counter.wasm and local TCP sockets"] +#[ignore = "requires v0.1.4 nx binary, built distributed_counter.wasm and local TCP sockets"] fn different_protocol_versions_reject_connection_without_exchanging_ops() { let wasm = assert_counter_wasm_exists(); let current_addr = free_addr(); let previous_addr = free_addr(); - let current_data = temp_path("protocol-v3"); - let previous_data = temp_path("protocol-v2"); + let current_data = temp_path("protocol-v5"); + let previous_data = temp_path("protocol-v4"); let current_nx = nx_bin(); let previous_nx = previous_nx_bin(); @@ -748,14 +752,14 @@ fn different_protocol_versions_reject_connection_without_exchanging_ops() { let previous_stdout = String::from_utf8_lossy(&previous_output.stdout); let previous_stderr = String::from_utf8_lossy(&previous_output.stderr); let protocol_mismatch_reported = current_stdout - .contains("protocol version mismatch: expected 4, got 2") - || current_stdout.contains("protocol version mismatch: expected 2, got 4") - || current_stderr.contains("protocol version mismatch: expected 2, got 4") - || current_stderr.contains("protocol version mismatch: expected 4, got 2") - || previous_stdout.contains("protocol version mismatch: expected 4, got 2") - || previous_stdout.contains("protocol version mismatch: expected 2, got 4") - || previous_stderr.contains("protocol version mismatch: expected 4, got 2") - || previous_stderr.contains("protocol version mismatch: expected 2, got 4"); + .contains("protocol version mismatch: expected 5, got 4") + || current_stdout.contains("protocol version mismatch: expected 4, got 5") + || current_stderr.contains("protocol version mismatch: expected 4, got 5") + || current_stderr.contains("protocol version mismatch: expected 5, got 4") + || previous_stdout.contains("protocol version mismatch: expected 5, got 4") + || previous_stdout.contains("protocol version mismatch: expected 4, got 5") + || previous_stderr.contains("protocol version mismatch: expected 5, got 4") + || previous_stderr.contains("protocol version mismatch: expected 4, got 5"); assert!( protocol_mismatch_reported, "neither node reported the protocol mismatch\ncurrent stdout:\n{current_stdout}\ncurrent stderr:\n{current_stderr}\nprevious stdout:\n{previous_stdout}\nprevious stderr:\n{previous_stderr}" diff --git a/crates/nx-cli/tests/support/discovery_lan.rs b/crates/nx-cli/tests/support/discovery_lan.rs new file mode 100644 index 0000000..4a28464 --- /dev/null +++ b/crates/nx-cli/tests/support/discovery_lan.rs @@ -0,0 +1,410 @@ +//! Real multicast, TCP, WASM and HTTP; three processes on ONE host, not three devices. +use super::{management_request, nx_bin, response_body, send_signal, temp_path, workspace_root}; +use std::collections::BTreeSet; +use std::fs::{self, File}; +use std::io::Read; +use std::net::{Ipv4Addr, SocketAddr, TcpListener, TcpStream}; +use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +const POLL: Duration = Duration::from_millis(100); +const WAIT: Duration = Duration::from_secs(90); +const SNAPSHOT_PATH: &str = "/api/v1/keys/ZGlzY292ZXJ5LWxhbg"; +// Retention is in operation COUNTS, not seconds. The scenario produces six ops. +const RETAINED_OPS: usize = 128; + +struct TestDirectory(PathBuf); + +impl TestDirectory { + fn new() -> Self { + let path = temp_path("mdns-three-daemons"); + fs::DirBuilder::new().mode(0o700).create(&path).unwrap(); + Self(path) + } +} + +impl Drop for TestDirectory { + fn drop(&mut self) { + // Only this run's exclusively created temporary directory is removed. + let _ = fs::remove_dir_all(&self.0); + } +} + +struct Daemon { + child: Option, + config: PathBuf, + log: PathBuf, + management: SocketAddr, + authorization: String, + reader: String, + writer: String, +} + +impl Drop for Daemon { + fn drop(&mut self) { + if let Some(mut child) = self.child.take() { + let _ = child.kill(); + let _ = child.wait(); + } + } +} + +impl Daemon { + fn start(&mut self) { + assert!(self.child.is_none()); + let output = File::options() + .create(true) + .append(true) + .mode(0o600) + .open(&self.log) + .unwrap(); + let mut command = Command::new(nx_bin()); + // A developer's NX_* settings must not inject static peers or disable auth. + for (name, _) in std::env::vars_os() { + if name.to_string_lossy().starts_with("NX_") { + command.env_remove(name); + } + } + let rust_log = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_string()); + self.child = Some( + command + .args(["serve", "--config"]) + .arg(&self.config) + .env("RUST_LOG", rust_log) + .stdin(Stdio::null()) + .stderr(output.try_clone().unwrap()) + .stdout(output) + .spawn() + .expect("spawn mDNS daemon"), + ); + self.wait("authenticated management readiness", |node| { + if TcpStream::connect_timeout(&node.management, Duration::from_millis(100)).is_err() { + return false; + } + node.request("GET", "/api/v1/ready", None, &[]) + .starts_with("HTTP/1.1 200 ") + }); + let denied = management_request(self.management, "GET", "/api/v1/health", None, None, &[]); + assert!( + denied.starts_with("HTTP/1.1 401 "), + "unauthenticated request was not denied" + ); + } + + fn assert_alive(&mut self) { + let status = self + .child + .as_mut() + .expect("running daemon") + .try_wait() + .unwrap(); + assert!( + status.is_none(), + "daemon exited: {status:?}\n{}", + self.logs() + ); + } + + fn logs(&self) -> String { + // Tokens never go in CLI arguments; redact defensively before diagnostics. + fs::read_to_string(&self.log).unwrap_or_default().replace( + self.authorization.trim_start_matches("Bearer "), + "[REDACTED]", + ) + } + + fn wait(&mut self, label: &str, mut condition: impl FnMut(&Self) -> bool) { + let deadline = Instant::now() + WAIT; + loop { + self.assert_alive(); + if condition(self) { + return; + } + assert!( + Instant::now() < deadline, + "timed out: {label}\n{}", + self.logs() + ); + std::thread::sleep(POLL); + } + } + + fn request(&self, method: &str, path: &str, content_type: Option<&str>, body: &[u8]) -> String { + management_request( + self.management, + method, + path, + Some(&self.authorization), + content_type, + body, + ) + } + + fn register(&self, wasm: &[u8]) -> String { + let response = self.request("POST", "/api/v1/modules", Some("application/wasm"), wasm); + assert!( + response.starts_with("HTTP/1.1 201 ") || response.starts_with("HTTP/1.1 200 "), + "register guest: {response}" + ); + let body: serde_json::Value = serde_json::from_str(response_body(&response)).unwrap(); + body["id"].as_str().unwrap().to_owned() + } + + fn register_guests(&mut self, reader: &[u8], writer: &[u8]) { + self.reader = self.register(reader); + self.writer = self.register(writer); + assert_ne!( + self.reader, self.writer, + "reader and writer must be distinct builds" + ); + } + + fn run(&self, module: &str) { + let response = self.request("POST", &format!("/api/v1/modules/{module}/runs"), None, &[]); + assert!( + response.starts_with("HTTP/1.1 204 "), + "guest execution failed: {response}" + ); + } + + fn persisted_snapshot(&self) -> (String, u64) { + let response = self.request("GET", SNAPSHOT_PATH, None, &[]); + assert!( + response.starts_with("HTTP/1.1 200 "), + "read snapshot: {response}" + ); + let (id, value) = response_body(&response) + .split_once('\n') + .expect("snapshot id and value"); + assert!(!id.is_empty()); + ( + id.to_owned(), + value.parse().expect("decimal counter snapshot"), + ) + } + + fn snapshot(&self) -> (String, u64) { + self.run(&self.reader); + self.persisted_snapshot() + } + + fn peer_ids(&self) -> BTreeSet { + let response = self.request("GET", "/api/v1/peers?limit=10", None, &[]); + assert!( + response.starts_with("HTTP/1.1 200 "), + "read peers: {response}" + ); + let body: serde_json::Value = serde_json::from_str(response_body(&response)).unwrap(); + assert!(body["next_cursor"].is_null(), "unexpected extra peers"); + let items = body["items"].as_array().unwrap(); + // The API lists connection addresses, not unique identities: symmetric + // dialing can leave both an inbound and an outbound link to one node. + let ids: BTreeSet<_> = items + .iter() + .map(|peer| peer["node_id"].as_str().unwrap().to_owned()) + .collect(); + let addresses: BTreeSet<_> = items + .iter() + .map(|peer| peer["address"].as_str().unwrap()) + .collect(); + assert_eq!( + addresses.len(), + items.len(), + "duplicate connection addresses" + ); + ids + } + + fn stop(&mut self) { + self.assert_alive(); + send_signal(self.child.as_ref().unwrap().id(), "TERM"); + let deadline = Instant::now() + Duration::from_secs(15); + loop { + if let Some(status) = self.child.as_mut().unwrap().try_wait().unwrap() { + assert!( + status.success(), + "daemon shutdown failed: {status}\n{}", + self.logs() + ); + self.child.take(); + assert!( + TcpStream::connect(self.management).is_err(), + "management listener still open" + ); + return; + } + assert!( + Instant::now() < deadline, + "daemon shutdown timed out\n{}", + self.logs() + ); + std::thread::sleep(POLL); + } + } +} + +fn wasm(mode: &str) -> Vec { + let path = workspace_root().join(format!( + "examples/discovery_lan/target/{mode}/wasm32-unknown-unknown/release/discovery_lan.wasm" + )); + fs::read(&path).unwrap_or_else(|error| { + panic!("build both discovery_lan guest variants first; missing {path:?}: {error}") + }) +} + +fn write_private(path: &Path, bytes: &[u8]) { + use std::io::Write; + File::options() + .write(true) + .create_new(true) + .mode(0o600) + .open(path) + .unwrap() + .write_all(bytes) + .unwrap(); +} + +// This test requires a physical or dedicated multicast-capable network interface. +// On virtualized single-host runners (e.g. cloud CI on macOS), running three separate +// processes binding UDP 5353 with SO_REUSEPORT can experience kernel-level packet +// load-balancing rather than full multicast fan-out to all sockets, causing intermittent +// peer discovery timeouts. It is intended for manual LAN verification or physical hosts. +#[test] +#[ignore = "requires NUMAX_MDNS_E2E=1, NUMAX_MDNS_LAN_IP, real multicast and both discovery_lan WASM builds"] +fn mdns_three_daemons_recover_missed_crdt_ops_after_restart() { + assert_eq!( + std::env::var("NUMAX_MDNS_E2E").as_deref(), + Ok("1"), + "explicit multicast opt-in required" + ); + let lan: Ipv4Addr = std::env::var("NUMAX_MDNS_LAN_IP") + .expect("set NUMAX_MDNS_LAN_IP to a real local LAN interface IPv4 address") + .parse() + .expect("LAN IPv4 address"); + assert!( + !lan.is_loopback() && !lan.is_unspecified() && !lan.is_multicast() && !lan.is_broadcast() + ); + let reader = wasm("reader"); + let writer = wasm("writer"); + let directory = TestDirectory::new(); + let cluster = directory.0.file_name().unwrap().to_str().unwrap(); + let mut nodes = Vec::new(); + // Hold all reservations until their daemon starts, avoiding duplicate ephemeral ports. + let mut reservations = Vec::new(); + for index in 0..3 { + let network = TcpListener::bind((lan, 0)).expect("bind real LAN interface"); + let management = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = network.local_addr().unwrap(); + let management_addr = management.local_addr().unwrap(); + let mut entropy = [0u8; 32]; + File::open("/dev/urandom") + .unwrap() + .read_exact(&mut entropy) + .unwrap(); + let token: String = entropy.iter().map(|byte| format!("{byte:02x}")).collect(); + let token_path = directory.0.join(format!("{index}.token")); + write_private(&token_path, token.as_bytes()); + let config = directory.0.join(format!("{index}.toml")); + let text = format!( + "[network]\nlisten = {endpoint:?}\npeers = []\n\ + [storage]\ndatastore_path = {data:?}\n\ + [management]\nlisten = {management:?}\ntoken_file = {token:?}\nallow_non_loopback = false\n\ + [discovery]\nmode = \"mdns\"\ncluster_id = {cluster:?}\ninstance_name = \"{cluster}-{index}\"\nadvertised_endpoint = {endpoint:?}\nmax_candidates = 8\nmax_instances = 8\n\ + [limits]\nmax_peers = 4\nqueued_ops_limit = 128\nop_log_limit = {RETAINED_OPS}\nseen_ops_limit = {RETAINED_OPS}\nanti_entropy_interval = \"200ms\"\nreconnect_initial_delay = \"100ms\"\nreconnect_max_delay = \"1s\"\n", + endpoint = endpoint.to_string(), + management = management_addr.to_string(), + data = directory.0.join(format!("data-{index}")).to_str().unwrap(), + token = token_path.to_str().unwrap(), + ); + write_private(&config, text.as_bytes()); + nodes.push(Daemon { + child: None, + config, + log: directory.0.join(format!("{index}.log")), + management: management_addr, + authorization: format!("Bearer {token}"), + reader: String::new(), + writer: String::new(), + }); + reservations.push((network, management)); + } + for (node, reservation) in nodes.iter_mut().zip(reservations) { + drop(reservation); + node.start(); + node.register_guests(&reader, &writer); + } + let identities: Vec<_> = nodes + .iter() + .map(|node| { + let (id, value) = node.snapshot(); + assert_eq!(value, 0, "fresh datastore must start empty"); + id + }) + .collect(); + let all_ids: BTreeSet<_> = identities.iter().cloned().collect(); + assert_eq!(all_ids.len(), 3); + for (node, id) in nodes.iter_mut().zip(&identities) { + let expected: BTreeSet<_> = all_ids + .iter() + .filter(|other| *other != id) + .cloned() + .collect(); + node.wait("discover the other two identities without --peer", |node| { + node.peer_ids() == expected + }); + } + eprintln!("mDNS: three daemons on one host discovered each other on {lan}"); + for node in &nodes { + node.run(&node.writer); + } + for node in &mut nodes { + node.wait("initial CRDT convergence to 3", |node| { + node.snapshot().1 == 3 + }); + } + nodes[2].stop(); + for index in 0..2 { + let expected = BTreeSet::from([identities[1 - index].clone()]); + nodes[index].wait("offline node removed from active connections", |node| { + node.peer_ids() == expected + }); + nodes[index].run(&nodes[index].writer); + } + for node in &mut nodes[..2] { + node.wait( + "survivors converge to 5 while third process is stopped", + |node| node.snapshot().1 == 5, + ); + } + nodes[2].start(); + // Read the old local observation BEFORE running the reader: this proves KV durability. + assert_eq!(nodes[2].persisted_snapshot(), (identities[2].clone(), 3)); + nodes[2].register_guests(&reader, &writer); + for (node, id) in nodes.iter_mut().zip(&identities) { + let expected: BTreeSet<_> = all_ids + .iter() + .filter(|other| *other != id) + .cloned() + .collect(); + node.wait("same identities rediscovered after restart", |node| { + node.peer_ids() == expected + }); + node.wait("missed-op recovery to 5 within 128-op retention", |node| { + node.snapshot() == (id.clone(), 5) + }); + } + nodes[2].run(&nodes[2].writer); + for (node, id) in nodes.iter_mut().zip(&identities) { + node.wait("restarted node can write; final convergence to 6", |node| { + node.snapshot() == (id.clone(), 6) + }); + } + for node in &mut nodes { + node.stop(); + } + eprintln!( + "mDNS E2E passed: 0 -> 3 -> offline writes -> 5 -> restart recovery -> 6; stable identities; retention {RETAINED_OPS} ops; clean shutdown" + ); +} diff --git a/crates/nx-core/Cargo.toml b/crates/nx-core/Cargo.toml index ddd9a93..0eca748 100644 --- a/crates/nx-core/Cargo.toml +++ b/crates/nx-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nx-core" -version = "0.1.4" +version = "0.1.5" edition = "2024" license.workspace = true @@ -13,14 +13,16 @@ anyhow = "1" async-trait = "0.1" blake3 = "1" getrandom = "0.4" +hickory-resolver = { version = "0.26", default-features = false, features = ["system-config", "tokio"] } +mdns-sd = { version = "0.21", default-features = false, features = ["async"] } sha2 = "0.11.0" serde_json = "1" wasmtime = "47.0.4" wasmtime-wasi = "47.0.4" -nx-store = { version = "0.1.4", path = "../nx-store" } -nx-sync = { version = "0.1.4", path = "../nx-sync" } -nx-net = { version = "0.1.4", path = "../nx-net" } -tokio = { version = "1", features = ["io-util", "net", "signal", "sync", "time"] } +nx-store = { version = "0.1.5", path = "../nx-store" } +nx-sync = { version = "0.1.5", path = "../nx-sync" } +nx-net = { version = "0.1.5", path = "../nx-net" } +tokio = { version = "1", features = ["fs", "io-util", "net", "signal", "sync", "time"] } tracing = "0.1" [target.'cfg(target_os = "linux")'.dependencies] @@ -29,7 +31,7 @@ inferno = { version = "0.12.8", optional = true, default-features = false } pprof = { version = "0.15", optional = true, default-features = false } [dev-dependencies] -nx-store = { version = "0.1.4", path = "../nx-store", features = ["test-utils"] } +nx-store = { version = "0.1.5", path = "../nx-store", features = ["test-utils"] } tempfile = "3" tokio = { version = "1", features = ["rt-multi-thread", "time"] } diff --git a/crates/nx-core/src/discovery.rs b/crates/nx-core/src/discovery.rs new file mode 100644 index 0000000..0c9179a --- /dev/null +++ b/crates/nx-core/src/discovery.rs @@ -0,0 +1,976 @@ +use std::error::Error; +use std::fmt; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use nx_net::BootstrapClientConfig; +use nx_sync::NodeId; +use tokio::sync::broadcast; + +use crate::SyncConfig; + +mod bootstrap_gossip; +mod dns_srv; +mod dynamic; +mod file_watch; +mod mdns; + +pub use bootstrap_gossip::{BootstrapGossipDiscovery, BootstrapGossipDiscoveryConfig}; +pub use dns_srv::{DnsSrvDiscovery, DnsSrvDiscoveryConfig}; +pub(crate) use dynamic::AbortOnDropTask; +pub use file_watch::{FileWatchDiscovery, FileWatchDiscoveryConfig}; +pub use mdns::{MdnsDiscovery, MdnsDiscoveryConfig}; + +/// Default number of discovery events retained for each provider watch channel. +pub const DEFAULT_DISCOVERY_EVENT_CAPACITY: usize = 128; + +/// Maximum configurable capacity of a discovery provider's event channel. +pub const MAX_DISCOVERY_EVENT_CAPACITY: usize = 4096; + +fn validate_event_capacity(provider: &str, event_capacity: usize) -> Result<(), DiscoveryError> { + if !(1..=MAX_DISCOVERY_EVENT_CAPACITY).contains(&event_capacity) { + return Err(DiscoveryError::InvalidConfiguration { + provider: provider.to_string(), + message: format!("event_capacity must be in 1..={MAX_DISCOVERY_EVENT_CAPACITY}"), + }); + } + Ok(()) +} + +/// Default maximum number of peer candidates retained by the coordinator. +pub const DEFAULT_MAX_PEER_CANDIDATES: usize = 1024; + +/// Default logical cluster used when no explicit discovery scope is supplied. +pub const DEFAULT_DISCOVERY_CLUSTER: &str = "default"; + +/// Resolved discovery configuration used when constructing a runtime. +#[derive(Debug, Clone)] +pub struct RuntimeDiscoveryConfig { + pub cluster_id: String, + pub advertised_endpoint: Option, + pub max_candidates: usize, + pub mode: RuntimeDiscoveryMode, +} + +impl Default for RuntimeDiscoveryConfig { + fn default() -> Self { + Self { + cluster_id: DEFAULT_DISCOVERY_CLUSTER.to_string(), + advertised_endpoint: None, + max_candidates: DEFAULT_MAX_PEER_CANDIDATES, + mode: RuntimeDiscoveryMode::Static, + } + } +} + +/// Provider-specific discovery configuration after precedence resolution. +#[derive(Debug, Clone)] +pub enum RuntimeDiscoveryMode { + Static, + Bootstrap(BootstrapDiscoverySettings), + Mdns(MdnsDiscoverySettings), + DnsSrv(DnsSrvDiscoverySettings), + File(FileDiscoverySettings), +} + +#[derive(Debug, Clone)] +pub struct BootstrapDiscoverySettings { + pub seeds: Vec, + pub refresh_interval: Duration, + pub retry_initial: Duration, + pub retry_max: Duration, + pub stale_after: Duration, + pub max_seeds: usize, +} + +impl BootstrapDiscoverySettings { + pub fn new(seeds: Vec) -> Self { + let defaults = BootstrapGossipDiscoveryConfig::new(seeds.clone()); + Self { + seeds, + refresh_interval: defaults.refresh_interval, + retry_initial: defaults.retry_initial, + retry_max: defaults.retry_max, + stale_after: defaults.stale_after, + max_seeds: defaults.max_seeds, + } + } +} + +#[derive(Debug, Clone)] +pub struct MdnsDiscoverySettings { + pub instance_name: String, + pub max_instances: usize, +} + +impl MdnsDiscoverySettings { + pub fn new(instance_name: impl Into) -> Self { + let instance_name = instance_name.into(); + let defaults = MdnsDiscoveryConfig::new(instance_name.clone()); + Self { + instance_name, + max_instances: defaults.max_instances, + } + } +} + +#[derive(Debug, Clone)] +pub struct DnsSrvDiscoverySettings { + pub service_name: String, + pub retry_interval: Duration, + pub max_refresh_interval: Duration, +} + +impl DnsSrvDiscoverySettings { + pub fn new(service_name: impl Into) -> Self { + let service_name = service_name.into(); + let defaults = DnsSrvDiscoveryConfig::new(service_name.clone()); + Self { + service_name, + retry_interval: defaults.retry_interval, + max_refresh_interval: defaults.max_refresh_interval, + } + } +} + +#[derive(Debug, Clone)] +pub struct FileDiscoverySettings { + pub path: PathBuf, + pub poll_interval: Duration, + pub max_file_bytes: usize, +} + +impl FileDiscoverySettings { + pub fn new(path: impl Into) -> Self { + let path = path.into(); + let defaults = FileWatchDiscoveryConfig::new(path.clone()); + Self { + path, + poll_interval: defaults.poll_interval, + max_file_bytes: defaults.max_file_bytes, + } + } +} + +pub(crate) fn build_runtime_discovery( + node_id: &NodeId, + sync: &SyncConfig, + config: &RuntimeDiscoveryConfig, +) -> Result<(Vec, DiscoveryRuntimeConfig), DiscoveryError> { + let mut providers = Vec::new(); + if matches!(config.mode, RuntimeDiscoveryMode::Static) || !sync.peers.is_empty() { + providers.push(DiscoveryProvider::new( + "static", + Arc::new(StaticDiscovery::new(sync.peers.clone())), + )); + } + + match &config.mode { + RuntimeDiscoveryMode::Static => {} + RuntimeDiscoveryMode::Bootstrap(settings) => { + let mut provider_config = BootstrapGossipDiscoveryConfig::new(settings.seeds.clone()); + provider_config.cluster_id = config.cluster_id.clone(); + provider_config.refresh_interval = settings.refresh_interval; + provider_config.retry_initial = settings.retry_initial; + provider_config.retry_max = settings.retry_max; + provider_config.stale_after = settings.stale_after; + provider_config.max_seeds = settings.max_seeds; + provider_config.max_candidates = config.max_candidates; + let mut client_config = BootstrapClientConfig::new(node_id.clone()); + client_config.tls = sync.tls.clone(); + client_config.max_message_size = sync.max_message_size; + client_config.socket_timeout = sync.socket_timeout; + client_config.serialization_format = sync.serialization_format; + client_config.max_response_candidates = config.max_candidates; + providers.push(DiscoveryProvider::new( + "bootstrap", + Arc::new(BootstrapGossipDiscovery::new( + provider_config, + client_config, + )?), + )); + } + RuntimeDiscoveryMode::Mdns(settings) => { + let mut provider_config = MdnsDiscoveryConfig::new(&settings.instance_name); + provider_config.cluster_id = config.cluster_id.clone(); + provider_config.max_instances = settings.max_instances; + provider_config.max_candidates = config.max_candidates; + providers.push(DiscoveryProvider::new( + "mdns", + Arc::new(MdnsDiscovery::new(provider_config)?), + )); + } + RuntimeDiscoveryMode::DnsSrv(settings) => { + let mut provider_config = DnsSrvDiscoveryConfig::new(&settings.service_name); + provider_config.cluster_id = config.cluster_id.clone(); + provider_config.retry_interval = settings.retry_interval; + provider_config.max_refresh_interval = settings.max_refresh_interval; + provider_config.max_candidates = config.max_candidates; + providers.push(DiscoveryProvider::new( + "dns-srv", + Arc::new(DnsSrvDiscovery::new(provider_config)?), + )); + } + RuntimeDiscoveryMode::File(settings) => { + let mut provider_config = FileWatchDiscoveryConfig::new(&settings.path); + provider_config.cluster_id = config.cluster_id.clone(); + provider_config.poll_interval = settings.poll_interval; + provider_config.max_file_bytes = settings.max_file_bytes; + provider_config.max_candidates = config.max_candidates; + providers.push(DiscoveryProvider::new( + "file", + Arc::new(FileWatchDiscovery::new(provider_config)?), + )); + } + } + + let mut runtime = DiscoveryRuntimeConfig::new() + .with_cluster_id(&config.cluster_id) + .with_max_candidates(config.max_candidates); + if let Some(endpoint) = &config.advertised_endpoint { + runtime = runtime.with_advertised_endpoint(endpoint); + } + Ok((providers, runtime)) +} + +/// Whether a provider can publish the local advertised endpoint. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AnnouncementSupport { + Unsupported, + Optional, + Required, +} + +/// Runtime policy shared by all discovery sources for one node. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DiscoveryRuntimeConfig { + cluster_id: String, + advertised_endpoint: Option, + max_candidates: usize, +} + +impl Default for DiscoveryRuntimeConfig { + fn default() -> Self { + Self { + cluster_id: DEFAULT_DISCOVERY_CLUSTER.to_string(), + advertised_endpoint: None, + max_candidates: DEFAULT_MAX_PEER_CANDIDATES, + } + } +} + +impl DiscoveryRuntimeConfig { + pub fn new() -> Self { + Self::default() + } + + pub fn with_cluster_id(mut self, cluster_id: impl Into) -> Self { + self.cluster_id = cluster_id.into(); + self + } + + pub fn with_advertised_endpoint(mut self, endpoint: impl Into) -> Self { + self.advertised_endpoint = Some(endpoint.into()); + self + } + + pub fn with_max_candidates(mut self, max_candidates: usize) -> Self { + self.max_candidates = max_candidates; + self + } + + pub fn cluster_id(&self) -> &str { + &self.cluster_id + } + + pub fn advertised_endpoint(&self) -> Option<&str> { + self.advertised_endpoint.as_deref() + } + + pub fn max_candidates(&self) -> usize { + self.max_candidates + } +} + +/// A named provider contribution and its optional candidate lease duration. +#[derive(Clone)] +pub struct DiscoveryProvider { + source_id: String, + provider: Arc, + candidate_ttl: Option, +} + +impl fmt::Debug for DiscoveryProvider { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("DiscoveryProvider") + .field("source_id", &self.source_id) + .field("cluster_id", &self.provider.cluster_id()) + .field("candidate_ttl", &self.candidate_ttl) + .finish_non_exhaustive() + } +} + +impl DiscoveryProvider { + pub fn new(source_id: impl Into, provider: Arc) -> Self { + Self { + source_id: source_id.into(), + provider, + candidate_ttl: None, + } + } + + pub fn with_candidate_ttl(mut self, candidate_ttl: Duration) -> Self { + self.candidate_ttl = Some(candidate_ttl); + self + } + + pub fn source_id(&self) -> &str { + &self.source_id + } + + pub fn provider(&self) -> &Arc { + &self.provider + } + + pub fn candidate_ttl(&self) -> Option { + self.candidate_ttl + } +} + +/// A complete provider view at one logical revision. +/// +/// Revisions are contiguous within a watch. A snapshot returned by +/// [`PeerDiscovery::watch`] is atomic with the event subscription: its first +/// event has revision `snapshot.revision() + 1`, and every later event advances +/// it by one. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DiscoverySnapshot { + revision: u64, + peers: Vec, + observations: Option>, +} + +impl DiscoverySnapshot { + pub fn new(revision: u64, peers: Vec) -> Self { + Self { + revision, + peers, + observations: None, + } + } + + /// Capture endpoint observation times, not cache publication times. A fresh + /// watch must preserve these times so resubscription cannot extend a lease. + pub fn observed(revision: u64, peers: Vec<(String, std::time::Instant)>) -> Self { + let (peers, observations) = peers.into_iter().unzip(); + Self { + revision, + peers, + observations: Some(observations), + } + } + + pub fn observations(&self) -> Option<&[std::time::Instant]> { + self.observations.as_deref() + } + + pub fn revision(&self) -> u64 { + self.revision + } + + pub fn peers(&self) -> &[String] { + &self.peers + } + + pub fn into_peers(self) -> Vec { + self.peers + } +} + +/// A change occurring after the snapshot associated with a watch. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DiscoveryEvent { + pub revision: u64, + pub change: DiscoveryChange, +} + +/// A change to the provider's peer candidates. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DiscoveryChange { + Added(String), + Removed(String), + /// Atomically replace the provider's complete ordered contribution. + Replaced(Vec), + /// Complete view with original per-endpoint observation times. Identical + /// peers with newer observations renew leases; cached republication does not. + Observed(DiscoverySnapshot), +} + +#[cfg(test)] +fn observed_peers(change: DiscoveryChange) -> Vec { + match change { + DiscoveryChange::Observed(snapshot) => { + assert!(snapshot.observations().is_some()); + snapshot.into_peers() + } + other => panic!("expected an observed snapshot, got {other:?}"), + } +} + +#[cfg(test)] +async fn next_changed_peers(watch: &mut DiscoveryWatch, previous: &[String]) -> Vec { + tokio::time::timeout(Duration::from_secs(2), async { + loop { + let peers = observed_peers(watch.recv().await.unwrap().change); + if peers != previous { + return peers; + } + } + }) + .await + .unwrap() +} + +/// The endpoint a provider is asked to announce. +/// +/// An announcement advertises only a connection candidate. It does not assert +/// peer identity, cluster membership, or authorization. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PeerAnnouncement { + pub endpoint: String, +} + +/// Errors exposed by discovery providers and watch delivery. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum DiscoveryError { + InvalidConfiguration { + provider: String, + message: String, + }, + Provider { + provider: String, + message: String, + retryable: bool, + }, + Unsupported { + provider: String, + operation: &'static str, + }, + WatchOverflow { + missed: u64, + }, + WatchRevision { + previous: u64, + received: u64, + }, + WatchInvalidated, + WatchClosed, +} + +impl fmt::Display for DiscoveryError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidConfiguration { provider, message } => { + write!( + formatter, + "{provider} discovery configuration is invalid: {message}" + ) + } + Self::Provider { + provider, + message, + retryable, + } => write!( + formatter, + "{provider} discovery provider failed (retryable: {retryable}): {message}" + ), + Self::Unsupported { + provider, + operation, + } => write!( + formatter, + "{provider} discovery provider does not support {operation}" + ), + Self::WatchOverflow { missed } => { + write!(formatter, "discovery watch missed {missed} event(s)") + } + Self::WatchRevision { previous, received } => write!( + formatter, + "discovery watch received revision {received} after revision {previous}" + ), + Self::WatchInvalidated => { + formatter.write_str("discovery watch is invalidated; create a fresh watch") + } + Self::WatchClosed => formatter.write_str("discovery watch closed"), + } + } +} + +impl Error for DiscoveryError {} + +/// An atomic snapshot and its bounded stream of subsequent changes. +/// +/// A lagging consumer receives [`DiscoveryError::WatchOverflow`] rather than a +/// silently incomplete view and must create a fresh watch. Dropping this value +/// cancels the subscription synchronously; it never owns a background task. +#[derive(Debug)] +pub struct DiscoveryWatch { + snapshot: DiscoverySnapshot, + events: broadcast::Receiver, + last_revision: u64, + invalidated: bool, +} + +impl DiscoveryWatch { + /// Build a watch from an atomically captured snapshot and bounded receiver. + /// + /// Provider implementations must create the receiver and snapshot under + /// the same state synchronization boundary, subscribing first, so no + /// transition can occur between them unnoticed. + pub fn new(snapshot: DiscoverySnapshot, events: broadcast::Receiver) -> Self { + let last_revision = snapshot.revision(); + Self { + snapshot, + events, + last_revision, + invalidated: false, + } + } + + pub fn snapshot(&self) -> &DiscoverySnapshot { + &self.snapshot + } + + pub async fn recv(&mut self) -> Result { + if self.invalidated { + return Err(DiscoveryError::WatchInvalidated); + } + + match self.events.recv().await { + Ok(event) => { + if let DiscoveryChange::Observed(snapshot) = &event.change + && snapshot.revision() != event.revision + { + self.invalidated = true; + return Err(DiscoveryError::WatchRevision { + previous: self.last_revision, + received: snapshot.revision(), + }); + } + if self.last_revision.checked_add(1) != Some(event.revision) { + self.invalidated = true; + return Err(DiscoveryError::WatchRevision { + previous: self.last_revision, + received: event.revision, + }); + } + self.last_revision = event.revision; + Ok(event) + } + Err(broadcast::error::RecvError::Lagged(missed)) => { + self.invalidated = true; + Err(DiscoveryError::WatchOverflow { missed }) + } + Err(broadcast::error::RecvError::Closed) => Err(DiscoveryError::WatchClosed), + } + } +} + +/// Source of peer connection candidates. +/// +/// Discovery never authorizes a candidate. Every resulting connection still +/// passes through the existing transport limits, TLS/mTLS checks, allowlists, +/// and wire handshake. +#[async_trait] +pub trait PeerDiscovery: Send + Sync { + /// Logical cluster whose candidates and announcements this provider serves. + fn cluster_id(&self) -> &str { + DEFAULT_DISCOVERY_CLUSTER + } + + /// Declare whether startup must publish an advertised endpoint. + fn announcement_support(&self) -> AnnouncementSupport { + AnnouncementSupport::Unsupported + } + + /// Return the provider's complete view at one logical revision. + async fn discover(&self) -> Result; + + /// Advertise a local endpoint, or return [`DiscoveryError::Unsupported`]. + async fn announce(&self, announcement: &PeerAnnouncement) -> Result<(), DiscoveryError>; + + /// Atomically subscribe to changes and return the snapshot they follow. + /// + /// Delivery must be bounded. Providers must make overflow observable and + /// must not silently discard events. Dropping the returned watch cancels + /// that subscription. + async fn watch(&self) -> Result; + + /// Stop provider-owned work and withdraw announcements made by this node. + /// + /// This hook is synchronous so an owner can initiate cancellation before + /// awaiting unrelated shutdown work. Implementations with background work + /// must make repeated calls safe and return promptly. + fn request_shutdown(&self) {} + + /// Wait for provider-owned work to stop and complete bounded withdrawal. + async fn shutdown(&self) -> Result<(), DiscoveryError> { + self.request_shutdown(); + Ok(()) + } +} + +/// Backward-compatible discovery provider for explicitly configured peers. +/// +/// Static discovery intentionally preserves input order and duplicates. Peer +/// candidate deduplication and connection admission remain responsibilities of +/// the coordinator and networking path. +#[derive(Debug)] +pub struct StaticDiscovery { + peers: Vec, + event_tx: broadcast::Sender, +} + +impl StaticDiscovery { + pub fn new(peers: Vec) -> Self { + Self::with_event_capacity(peers, DEFAULT_DISCOVERY_EVENT_CAPACITY) + } + + /// Construct without validation errors, preserving peer order and duplicates. + /// + /// Capacity is clamped to `1..=MAX_DISCOVERY_EVENT_CAPACITY`: zero becomes + /// one and larger values become the maximum. For strict validation, use + /// [`Self::try_with_event_capacity`]. + pub fn with_event_capacity(peers: Vec, event_capacity: usize) -> Self { + let (event_tx, _) = + broadcast::channel(event_capacity.clamp(1, MAX_DISCOVERY_EVENT_CAPACITY)); + Self { peers, event_tx } + } + + /// Construct with capacity in `1..=MAX_DISCOVERY_EVENT_CAPACITY`. + /// + /// Returns [`DiscoveryError::InvalidConfiguration`] outside that range, + /// before allocating the channel. Peer order and duplicates are preserved. + pub fn try_with_event_capacity( + peers: Vec, + event_capacity: usize, + ) -> Result { + validate_event_capacity("static", event_capacity)?; + Ok(Self::with_event_capacity(peers, event_capacity)) + } + + fn snapshot(&self) -> DiscoverySnapshot { + DiscoverySnapshot::new(0, self.peers.clone()) + } +} + +#[async_trait] +impl PeerDiscovery for StaticDiscovery { + async fn discover(&self) -> Result { + Ok(self.snapshot()) + } + + async fn announce(&self, _announcement: &PeerAnnouncement) -> Result<(), DiscoveryError> { + Err(DiscoveryError::Unsupported { + provider: "static".to_string(), + operation: "announcement", + }) + } + + async fn watch(&self) -> Result { + // Subscribe before taking the snapshot. StaticDiscovery is immutable, + // while dynamic providers must use the same ordering under their state + // synchronization boundary to preserve this no-gap contract. + let events = self.event_tx.subscribe(); + Ok(DiscoveryWatch::new(self.snapshot(), events)) + } +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use super::*; + + fn assert_event_capacity_bounds( + provider: &str, + construct: impl Fn(usize) -> Result<(), DiscoveryError>, + ) { + for capacity in [0, MAX_DISCOVERY_EVENT_CAPACITY + 1, usize::MAX] { + assert_eq!( + construct(capacity), + Err(DiscoveryError::InvalidConfiguration { + provider: provider.to_string(), + message: format!( + "event_capacity must be in 1..={MAX_DISCOVERY_EVENT_CAPACITY}" + ), + }), + "{provider}: capacity {capacity} must be rejected", + ); + } + for capacity in [ + 1, + DEFAULT_DISCOVERY_EVENT_CAPACITY, + MAX_DISCOVERY_EVENT_CAPACITY, + ] { + assert_eq!( + construct(capacity), + Ok(()), + "{provider}: capacity {capacity}" + ); + } + } + + #[test] + fn bootstrap_event_capacity_bounds() { + assert_event_capacity_bounds("bootstrap", |capacity| { + let mut config = BootstrapGossipDiscoveryConfig::new(vec!["seed:9000".into()]); + config.event_capacity = capacity; + let mut client = BootstrapClientConfig::new(NodeId::new("client")); + client.max_response_candidates = config.max_candidates; + BootstrapGossipDiscovery::new(config, client).map(drop) + }); + } + + #[test] + fn mdns_event_capacity_bounds() { + assert_event_capacity_bounds("mdns", |capacity| { + let mut config = MdnsDiscoveryConfig::new("capacity-test"); + config.event_capacity = capacity; + MdnsDiscovery::new(config).map(drop) + }); + } + + #[test] + fn dns_srv_event_capacity_bounds() { + assert_event_capacity_bounds("dns-srv", |capacity| { + let mut config = DnsSrvDiscoveryConfig::new("_numax._tcp.example."); + config.event_capacity = capacity; + dns_srv::validate_config(&config) + }); + } + + #[test] + fn file_event_capacity_bounds() { + assert_event_capacity_bounds("file", |capacity| { + let mut config = FileWatchDiscoveryConfig::new("unused-capacity-test-peers"); + config.event_capacity = capacity; + FileWatchDiscovery::new(config).map(drop) + }); + } + + #[test] + fn static_try_event_capacity_bounds() { + let peers = vec![ + "peer-b:9000".into(), + "peer-a:9000".into(), + "peer-b:9000".into(), + ]; + assert_event_capacity_bounds("static", |capacity| { + StaticDiscovery::try_with_event_capacity(peers.clone(), capacity).map(|provider| { + assert_eq!(provider.snapshot().peers(), peers); + }) + }); + } + + #[tokio::test] + async fn static_legacy_event_capacity_normalizes_extremes_and_preserves_peers() { + let peers = vec![ + "peer-b:9000".into(), + "peer-a:9000".into(), + "peer-b:9000".into(), + ]; + for (capacity, normalized) in [ + (0, 1), + (1, 1), + (MAX_DISCOVERY_EVENT_CAPACITY, MAX_DISCOVERY_EVENT_CAPACITY), + ( + MAX_DISCOVERY_EVENT_CAPACITY + 1, + MAX_DISCOVERY_EVENT_CAPACITY, + ), + (usize::MAX, MAX_DISCOVERY_EVENT_CAPACITY), + ] { + let provider = StaticDiscovery::with_event_capacity(peers.clone(), capacity); + let mut events = provider.watch().await.unwrap(); + assert_eq!(provider.discover().await.unwrap().peers(), peers); + assert_eq!(events.snapshot().peers(), peers); + // Static providers never publish in production. Inject events here + // to verify the actual allocated channel bound, not just no panic. + for revision in 1..=normalized + 1 { + provider + .event_tx + .send(DiscoveryEvent { + revision: revision as u64, + change: DiscoveryChange::Replaced(Vec::new()), + }) + .unwrap(); + } + assert_eq!( + events.recv().await, + Err(DiscoveryError::WatchOverflow { missed: 1 }) + ); + } + } + + #[test] + fn runtime_factory_composes_explicit_peers_with_dynamic_discovery() { + let sync = SyncConfig::new() + .with_listen_addr("127.0.0.1:9000") + .with_peer("127.0.0.1:9001"); + let config = RuntimeDiscoveryConfig { + mode: RuntimeDiscoveryMode::Bootstrap(BootstrapDiscoverySettings::new(vec![ + "127.0.0.1:9100".to_string(), + ])), + ..RuntimeDiscoveryConfig::default() + }; + + let (providers, runtime) = + build_runtime_discovery(&NodeId::new("local"), &sync, &config).unwrap(); + + assert_eq!( + providers + .iter() + .map(DiscoveryProvider::source_id) + .collect::>(), + ["static", "bootstrap"] + ); + assert_eq!(runtime.cluster_id(), DEFAULT_DISCOVERY_CLUSTER); + assert_eq!(runtime.max_candidates(), DEFAULT_MAX_PEER_CANDIDATES); + } + + #[tokio::test] + async fn static_snapshot_preserves_order_and_duplicates() { + let discovery = StaticDiscovery::new(vec![ + "peer-b:9000".to_string(), + "peer-a:9000".to_string(), + "peer-b:9000".to_string(), + ]); + + let snapshot = discovery.discover().await.unwrap(); + + assert_eq!(snapshot.revision(), 0); + assert_eq!( + snapshot.peers(), + ["peer-b:9000", "peer-a:9000", "peer-b:9000"] + ); + } + + #[tokio::test] + async fn static_watch_snapshot_matches_discover_without_a_gap() { + let discovery = StaticDiscovery::new(vec!["peer-a:9000".to_string()]); + + let discovered = discovery.discover().await.unwrap(); + let watch = discovery.watch().await.unwrap(); + + assert_eq!(watch.snapshot(), &discovered); + } + + #[tokio::test] + async fn static_announcement_is_explicitly_unsupported() { + let discovery = StaticDiscovery::new(Vec::new()); + let announcement = PeerAnnouncement { + endpoint: "127.0.0.1:9000".to_string(), + }; + + let error = discovery.announce(&announcement).await.unwrap_err(); + + assert_eq!( + error, + DiscoveryError::Unsupported { + provider: "static".to_string(), + operation: "announcement", + } + ); + } + + #[tokio::test] + async fn lagging_watch_reports_overflow() { + let (event_tx, events) = broadcast::channel(1); + let mut watch = DiscoveryWatch::new(DiscoverySnapshot::new(0, Vec::new()), events); + let first = DiscoveryEvent { + revision: 1, + change: DiscoveryChange::Added("peer-a:9000".to_string()), + }; + let second = DiscoveryEvent { + revision: 2, + change: DiscoveryChange::Added("peer-b:9000".to_string()), + }; + event_tx.send(first).unwrap(); + event_tx.send(second).unwrap(); + + assert_eq!( + watch.recv().await.unwrap_err(), + DiscoveryError::WatchOverflow { missed: 1 } + ); + assert_eq!( + watch.recv().await.unwrap_err(), + DiscoveryError::WatchInvalidated + ); + } + + #[tokio::test] + async fn watch_accepts_contiguous_revisions() { + let (event_tx, events) = broadcast::channel(1); + let mut watch = DiscoveryWatch::new(DiscoverySnapshot::new(4, Vec::new()), events); + let expected = DiscoveryEvent { + revision: 5, + change: DiscoveryChange::Added("peer-a:9000".to_string()), + }; + event_tx.send(expected.clone()).unwrap(); + + assert_eq!(watch.recv().await.unwrap(), expected); + } + + #[tokio::test] + async fn watch_rejects_non_contiguous_revisions() { + let (event_tx, events) = broadcast::channel(1); + let mut watch = DiscoveryWatch::new(DiscoverySnapshot::new(4, Vec::new()), events); + event_tx + .send(DiscoveryEvent { + revision: 6, + change: DiscoveryChange::Added("peer-a:9000".to_string()), + }) + .unwrap(); + + assert_eq!( + watch.recv().await.unwrap_err(), + DiscoveryError::WatchRevision { + previous: 4, + received: 6, + } + ); + assert_eq!( + watch.recv().await.unwrap_err(), + DiscoveryError::WatchInvalidated + ); + } + + #[tokio::test] + async fn dropping_watch_cancels_subscription_without_a_task() { + let discovery = StaticDiscovery::new(Vec::new()); + let watch = discovery.watch().await.unwrap(); + assert_eq!(discovery.event_tx.receiver_count(), 1); + + drop(watch); + + assert_eq!(discovery.event_tx.receiver_count(), 0); + } + + #[tokio::test] + async fn watch_reports_provider_closure() { + let (event_tx, events) = broadcast::channel(1); + let mut watch = DiscoveryWatch::new(DiscoverySnapshot::new(0, Vec::new()), events); + drop(event_tx); + + assert_eq!(watch.recv().await.unwrap_err(), DiscoveryError::WatchClosed); + } + + #[test] + fn peer_discovery_is_object_safe() { + let discovery: Arc = Arc::new(StaticDiscovery::new(Vec::new())); + assert_eq!(Arc::strong_count(&discovery), 1); + } +} diff --git a/crates/nx-core/src/discovery/bootstrap_gossip.rs b/crates/nx-core/src/discovery/bootstrap_gossip.rs new file mode 100644 index 0000000..1f9a2b5 --- /dev/null +++ b/crates/nx-core/src/discovery/bootstrap_gossip.rs @@ -0,0 +1,1459 @@ +use std::collections::{HashMap, HashSet}; +use std::future::{Future, pending}; +use std::sync::{Arc, Mutex as StdMutex}; +use std::time::Duration; + +use async_trait::async_trait; +use nx_net::{ + BootstrapClient, BootstrapClientConfig, BootstrapError, BootstrapRequest, NetError, + WireRetryPolicy, +}; +use tokio::sync::watch; +use tokio::time::Instant; + +use super::dynamic::{DynamicState, ProviderTask, checked_deadline, validate_durations}; +use super::{ + AnnouncementSupport, DEFAULT_DISCOVERY_CLUSTER, DEFAULT_DISCOVERY_EVENT_CAPACITY, + DEFAULT_MAX_PEER_CANDIDATES, DiscoveryError, DiscoverySnapshot, DiscoveryWatch, + PeerAnnouncement, PeerDiscovery, validate_event_capacity, +}; + +const PROVIDER: &str = "bootstrap"; +const DEFAULT_REFRESH_INTERVAL: Duration = Duration::from_secs(20); +const DEFAULT_RETRY_INITIAL: Duration = Duration::from_millis(500); +const DEFAULT_RETRY_MAX: Duration = Duration::from_secs(30); +const DEFAULT_STALE_AFTER: Duration = Duration::from_secs(120); +const DEFAULT_MAX_SEEDS: usize = 32; +const SHUTDOWN_WITHDRAWAL_BUDGET: Duration = Duration::from_secs(4); + +/// Seed probing, retention, and delivery policy for bootstrap gossip. +#[derive(Debug, Clone)] +pub struct BootstrapGossipDiscoveryConfig { + pub seeds: Vec, + pub cluster_id: String, + pub refresh_interval: Duration, + pub retry_initial: Duration, + pub retry_max: Duration, + pub stale_after: Duration, + pub max_seeds: usize, + pub max_candidates: usize, + /// Event channel capacity in `1..=super::MAX_DISCOVERY_EVENT_CAPACITY`. + /// Defaults to [`DEFAULT_DISCOVERY_EVENT_CAPACITY`]; validated by the provider constructor. + pub event_capacity: usize, +} + +impl BootstrapGossipDiscoveryConfig { + pub fn new(seeds: Vec) -> Self { + Self { + seeds, + cluster_id: DEFAULT_DISCOVERY_CLUSTER.to_string(), + refresh_interval: DEFAULT_REFRESH_INTERVAL, + retry_initial: DEFAULT_RETRY_INITIAL, + retry_max: DEFAULT_RETRY_MAX, + stale_after: DEFAULT_STALE_AFTER, + max_seeds: DEFAULT_MAX_SEEDS, + max_candidates: DEFAULT_MAX_PEER_CANDIDATES, + event_capacity: DEFAULT_DISCOVERY_EVENT_CAPACITY, + } + } +} + +struct Lifecycle { + stopped: bool, + shutdown: Option>, + task: Option, +} + +struct Inner { + config: BootstrapGossipDiscoveryConfig, + client: BootstrapClient, + state: Arc, + announcement_tx: watch::Sender>, + announced_seeds: Arc>>, + lifecycle: StdMutex, +} + +impl Drop for Inner { + fn drop(&mut self) { + let lifecycle = self + .lifecycle + .get_mut() + .unwrap_or_else(|error| error.into_inner()); + if let Some(shutdown) = lifecycle.shutdown.take() { + let _ = shutdown.send(true); + } + } +} + +/// Learns bounded endpoint suggestions from authenticated bootstrap seeds. +/// +/// Authentication covers the seed that returned a response. Suggested +/// endpoints remain untrusted candidates and are authenticated independently +/// if the normal reconnection loop later dials them. +pub struct BootstrapGossipDiscovery { + inner: Arc, +} + +impl BootstrapGossipDiscovery { + pub fn new( + mut config: BootstrapGossipDiscoveryConfig, + client_config: BootstrapClientConfig, + ) -> Result { + validate_config(&config)?; + if config.max_candidates > client_config.max_response_candidates { + return Err(invalid(format!( + "max_candidates exceeds the bootstrap client response limit of {}", + client_config.max_response_candidates + ))); + } + let mut seen = HashSet::new(); + let mut seeds = Vec::with_capacity(config.seeds.len()); + for seed in &config.seeds { + let seed = crate::sync_manager::canonicalize_endpoint(seed) + .map_err(|error| invalid(format!("invalid bootstrap seed {seed:?}: {error}")))?; + if seen.insert(seed.clone()) { + seeds.push(seed); + } + } + config.seeds = seeds; + let client = BootstrapClient::new(client_config) + .map_err(|error| invalid(format!("invalid bootstrap client: {error}")))?; + let (announcement_tx, _) = watch::channel(None); + Ok(Self { + inner: Arc::new(Inner { + state: Arc::new(DynamicState::new(config.event_capacity)), + config, + client, + announcement_tx, + announced_seeds: Arc::new(StdMutex::new(HashSet::new())), + lifecycle: StdMutex::new(Lifecycle { + stopped: false, + shutdown: None, + task: None, + }), + }), + }) + } + + fn ensure_started(&self) -> Result<(), DiscoveryError> { + let mut lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + if lifecycle.stopped { + return Err(provider_error("provider is shut down", false)); + } + if let Some(task) = lifecycle.task.as_ref() + && task.running(PROVIDER, &self.inner.state)? + { + return Ok(()); + } + let (shutdown, shutdown_rx) = watch::channel(false); + let config = self.inner.config.clone(); + let client = self.inner.client.clone(); + let state = Arc::clone(&self.inner.state); + let announcement_rx = self.inner.announcement_tx.subscribe(); + let announced_seeds = Arc::clone(&self.inner.announced_seeds); + let mut cleanup = BootstrapCleanup { + client: self.inner.client.clone(), + cluster_id: self.inner.config.cluster_id.clone(), + announcement_tx: self.inner.announcement_tx.clone(), + announced_seeds: Arc::clone(&self.inner.announced_seeds), + state: Arc::clone(&self.inner.state), + preserve_announcement: true, + }; + let cleanup_shutdown = shutdown_rx.clone(); + lifecycle.shutdown = Some(shutdown); + lifecycle.task = Some(ProviderTask::spawn( + PROVIDER, + state.clone(), + shutdown_rx.clone(), + async move { + run_bootstrap( + config, + client, + state, + announcement_rx, + announced_seeds, + shutdown_rx, + ) + .await; + Ok(()) + }, + move || async move { + let result = cleanup.withdraw().await; + cleanup.preserve_announcement = + !*cleanup_shutdown.borrow() && cleanup_shutdown.has_changed().is_ok(); + drop(cleanup); + result + }, + )); + Ok(()) + } +} + +#[async_trait] +impl PeerDiscovery for BootstrapGossipDiscovery { + fn cluster_id(&self) -> &str { + &self.inner.config.cluster_id + } + + fn announcement_support(&self) -> AnnouncementSupport { + AnnouncementSupport::Required + } + + async fn discover(&self) -> Result { + self.ensure_started()?; + Ok(self.inner.state.snapshot()) + } + + async fn announce(&self, announcement: &PeerAnnouncement) -> Result<(), DiscoveryError> { + let lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + if lifecycle.stopped { + return Err(provider_error("provider is shut down", false)); + } + let endpoint = crate::sync_manager::canonicalize_endpoint(&announcement.endpoint) + .map_err(|error| provider_error(error.to_string(), false))?; + // Serialize publication with request_shutdown, not just its check. + self.inner.announcement_tx.send_replace(Some(endpoint)); + Ok(()) + } + + async fn watch(&self) -> Result { + self.ensure_started()?; + self.inner.state.live_watch() + } + + fn request_shutdown(&self) { + let mut lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + lifecycle.stopped = true; + if let Some(shutdown) = lifecycle.shutdown.as_ref() { + let _ = shutdown.send(true); + } + } + + async fn shutdown(&self) -> Result<(), DiscoveryError> { + self.request_shutdown(); + let task = { + let lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + lifecycle.task.clone() + }; + let result = match task { + Some(task) => task.join().await, + None => Ok(()), + }; + self.inner.announcement_tx.send_replace(None); + result + } +} + +struct BootstrapCleanup { + preserve_announcement: bool, + client: C, + cluster_id: String, + announcement_tx: watch::Sender>, + announced_seeds: Arc>>, + state: Arc, +} + +impl BootstrapCleanup { + async fn withdraw(&self) -> Result<(), DiscoveryError> { + if self.announcement_tx.borrow().is_some() { + let announced_seeds = self + .announced_seeds + .lock() + .unwrap_or_else(|error| error.into_inner()) + .iter() + .cloned() + .collect::>(); + let deadline = checked_deadline( + Instant::now(), + SHUTDOWN_WITHDRAWAL_BUDGET, + PROVIDER, + "withdrawal", + )?; + for (index, seed) in announced_seeds.iter().enumerate() { + let now = Instant::now(); + let remaining = deadline.saturating_duration_since(now); + if remaining.is_zero() { + break; + } + let remaining_seeds = u32::try_from(announced_seeds.len() - index) + .unwrap_or(u32::MAX) + .max(1); + let request = BootstrapRequest::new(self.cluster_id.clone(), 1); + let slot_deadline = checked_deadline( + now, + remaining / remaining_seeds, + PROVIDER, + "withdrawal slot", + )?; + match tokio::time::timeout_at(slot_deadline, self.client.query(seed, request)).await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => { + tracing::debug!(%error, %seed, "bootstrap announcement withdrawal failed"); + } + Err(_) => { + tracing::debug!(%seed, "bootstrap announcement withdrawal timed out"); + } + } + } + } + Ok(()) + } +} + +impl Drop for BootstrapCleanup { + fn drop(&mut self) { + self.announced_seeds + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clear(); + if !self.preserve_announcement { + self.announcement_tx.send_replace(None); + } + self.state.replace(Vec::new()); + } +} + +struct SeedView { + endpoints: Vec, + expires_at: Instant, + observed_at: std::time::Instant, +} + +struct SeedSchedule { + next_probe: Instant, + not_before: Instant, + retry_delay: Duration, + disabled: bool, +} + +impl SeedSchedule { + fn deadline(&self) -> Option { + (!self.disabled).then_some(self.next_probe.max(self.not_before)) + } + + fn announce(&mut self, now: Instant) { + self.next_probe = now; + } + + fn failed( + &mut self, + config: &BootstrapGossipDiscoveryConfig, + error: &BootstrapError, + now: Instant, + ) -> Result<(), DiscoveryError> { + self.disabled = bootstrap_error_is_fatal(error); + // Preserve the configured cap on server-requested backoff, but retain + // an absolute barrier independent of view expiry and announcements. + // Disable before checking: an unrepresentable barrier must never turn + // into an immediate retry, including after a new announcement. + let was_disabled = self.disabled; + self.disabled = true; + self.not_before = checked_deadline( + now, + bootstrap_retry_after(error) + .unwrap_or_default() + .min(config.retry_max), + PROVIDER, + "retry_after", + )?; + self.next_probe = checked_deadline(now, self.retry_delay, PROVIDER, "retry_delay")?; + self.retry_delay = self.retry_delay.saturating_mul(2).min(config.retry_max); + self.disabled = was_disabled; + Ok(()) + } +} + +#[async_trait] +trait SeedClient: Send + Sync { + async fn query( + &self, + seed: &str, + request: BootstrapRequest, + ) -> Result; +} + +#[async_trait] +impl SeedClient for BootstrapClient { + async fn query( + &self, + seed: &str, + request: BootstrapRequest, + ) -> Result { + BootstrapClient::query(self, seed, request).await + } +} + +async fn run_bootstrap( + config: BootstrapGossipDiscoveryConfig, + client: impl SeedClient, + state: Arc, + mut announcement_rx: watch::Receiver>, + announced_seeds: Arc>>, + mut shutdown_rx: watch::Receiver, +) { + let mut views = HashMap::::new(); + let now = Instant::now(); + let mut schedules: Vec<_> = config + .seeds + .iter() + .map(|_| SeedSchedule { + next_probe: now, + not_before: now, + retry_delay: config.retry_initial, + disabled: false, + }) + .collect(); + + loop { + if *shutdown_rx.borrow() || shutdown_rx.has_changed().is_err() { + return; + } + let announcement = announcement_rx.borrow_and_update().clone(); + for (seed, schedule) in config.seeds.iter().zip(&mut schedules) { + if schedule + .deadline() + .is_none_or(|deadline| deadline > Instant::now()) + { + continue; + } + let mut request = + BootstrapRequest::new(config.cluster_id.clone(), config.max_candidates); + if let Some(endpoint) = &announcement { + request = request.with_advertised_endpoint(endpoint.clone()); + // Sending may apply the announcement even when the response + // is lost, the query is cancelled, or decoding fails. + announced_seeds + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert(seed.clone()); + } + let Some(result) = await_query_with_expiry( + client.query(seed, request), + &config, + &mut views, + &state, + &mut shutdown_rx, + ) + .await + else { + return; + }; + match result { + Ok(response) => { + schedule.retry_delay = config.retry_initial; + let now = Instant::now(); + let deadlines = seed_deadlines(now, &config, response.candidate_ttl); + let (refresh, expires_at) = match deadlines { + Ok(deadlines) => deadlines, + Err(error) => { + tracing::error!(%error, %seed, "disabling bootstrap seed schedule"); + schedule.disabled = true; + views.remove(seed); + publish_views(&config, &mut views, &state); + continue; + } + }; + schedule.next_probe = refresh; + let mut endpoints = Vec::with_capacity(response.endpoints.len() + 1); + endpoints.push(seed.clone()); + for endpoint in response.endpoints { + if !endpoints.contains(&endpoint) { + endpoints.push(endpoint); + } + } + endpoints.truncate(config.max_candidates); + views.insert( + seed.clone(), + SeedView { + endpoints, + observed_at: std::time::Instant::now(), + expires_at, + }, + ); + } + Err(error) => { + if let Err(error) = schedule.failed(&config, &error, Instant::now()) { + tracing::error!(%error, %seed, "disabling bootstrap seed schedule"); + } + tracing::debug!(%error, %seed, "bootstrap seed query failed"); + } + } + publish_views(&config, &mut views, &state); + } + + publish_views(&config, &mut views, &state); + let next_expiry = views.values().map(|view| view.expires_at).min(); + let deadline = schedules + .iter() + .filter_map(SeedSchedule::deadline) + .chain(next_expiry) + .min(); + tokio::select! { + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + break; + } + } + changed = announcement_rx.changed() => { + if changed.is_err() { + break; + } + let now = Instant::now(); + for schedule in &mut schedules { schedule.announce(now); } + } + _ = sleep_until_optional(deadline) => {} + } + } +} + +fn seed_deadlines( + now: Instant, + config: &BootstrapGossipDiscoveryConfig, + candidate_ttl: Duration, +) -> Result<(Instant, Instant), DiscoveryError> { + let refresh = checked_deadline(now, config.refresh_interval, PROVIDER, "refresh_interval")?; + let expiry = checked_deadline( + now, + candidate_ttl.min(config.stale_after), + PROVIDER, + "candidate_ttl", + )?; + Ok((refresh, expiry)) +} + +async fn await_query_with_expiry( + query: F, + config: &BootstrapGossipDiscoveryConfig, + views: &mut HashMap, + state: &DynamicState, + shutdown: &mut watch::Receiver, +) -> Option +where + F: Future, +{ + tokio::pin!(query); + loop { + if *shutdown.borrow() || shutdown.has_changed().is_err() { + return None; + } + let next_expiry = views.values().map(|view| view.expires_at).min(); + tokio::select! { + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return None; + } + } + result = &mut query => return Some(result), + _ = sleep_until_optional(next_expiry) => { + publish_views(config, views, state); + } + } + } +} + +async fn sleep_until_optional(deadline: Option) { + match deadline { + Some(deadline) => tokio::time::sleep_until(deadline).await, + None => pending().await, + } +} + +fn publish_views( + config: &BootstrapGossipDiscoveryConfig, + views: &mut HashMap, + state: &DynamicState, +) { + let now = Instant::now(); + views.retain(|_, view| view.expires_at > now); + let peers = flatten_views(&config.seeds, views, config.max_candidates); + state.observe_at( + peers + .into_iter() + .filter_map(|peer| { + let observed_at = views + .values() + .filter(|view| view.endpoints.contains(&peer)) + .map(|view| view.observed_at) + .max()?; + Some((peer, observed_at)) + }) + .collect(), + ); +} + +fn flatten_views( + seeds: &[String], + views: &HashMap, + max_candidates: usize, +) -> Vec { + let mut result = Vec::new(); + for seed in seeds { + let Some(view) = views.get(seed) else { + continue; + }; + for endpoint in &view.endpoints { + if result.len() == max_candidates { + return result; + } + if !result.contains(endpoint) { + result.push(endpoint.clone()); + } + } + } + result +} + +fn bootstrap_error_is_fatal(error: &BootstrapError) -> bool { + matches!( + error, + BootstrapError::InvalidConfig(_) + | BootstrapError::Rejected { .. } + | BootstrapError::InvalidResponse(_) + | BootstrapError::NodeConfig(_) + ) || matches!( + error, + BootstrapError::Transport(NetError::Wire(wire)) + if matches!(wire.retry_policy(), WireRetryPolicy::Fatal | WireRetryPolicy::RequestFatal) + ) +} + +fn bootstrap_retry_after(error: &BootstrapError) -> Option { + match error { + BootstrapError::Transport(NetError::Wire(wire)) => match wire.retry_policy() { + WireRetryPolicy::RetryAfter(delay) => Some(delay), + _ => None, + }, + _ => None, + } +} + +fn validate_config(config: &BootstrapGossipDiscoveryConfig) -> Result<(), DiscoveryError> { + validate_event_capacity(PROVIDER, config.event_capacity)?; + if config.seeds.is_empty() { + return Err(invalid("at least one bootstrap seed is required")); + } + if config.seeds.len() > config.max_seeds { + return Err(invalid(format!( + "bootstrap seed count exceeds the {} seed limit", + config.max_seeds + ))); + } + if config.cluster_id.is_empty() || config.cluster_id.len() > 128 { + return Err(invalid("cluster_id length must be in 1..=128 bytes")); + } + if config.refresh_interval.is_zero() + || config.retry_initial.is_zero() + || config.retry_max < config.retry_initial + || config.stale_after.is_zero() + || config.max_seeds == 0 + || config.max_candidates == 0 + || config.max_candidates > nx_net::MAX_BOOTSTRAP_RESPONSE_CAPACITY + { + return Err(invalid("intervals and limits are inconsistent")); + } + validate_durations( + PROVIDER, + &[ + ("refresh_interval", config.refresh_interval), + ("retry_initial", config.retry_initial), + ("retry_max", config.retry_max), + ("stale_after", config.stale_after), + ], + ) +} + +fn invalid(message: impl Into) -> DiscoveryError { + DiscoveryError::InvalidConfiguration { + provider: PROVIDER.to_string(), + message: message.into(), + } +} + +fn provider_error(message: impl Into, retryable: bool) -> DiscoveryError { + DiscoveryError::Provider { + provider: PROVIDER.to_string(), + message: message.into(), + retryable, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use nx_net::{BootstrapServerConfig, Node, NodeConfig}; + use nx_sync::NodeId; + + #[test] + fn extreme_durations_are_rejected_before_starting() { + for field in [ + "refresh_interval", + "retry_initial", + "retry_max", + "stale_after", + ] { + let mut config = BootstrapGossipDiscoveryConfig::new(vec!["seed:9000".into()]); + match field { + "refresh_interval" => config.refresh_interval = Duration::MAX, + "retry_initial" => { + config.retry_initial = Duration::MAX; + config.retry_max = Duration::MAX; + } + "retry_max" => config.retry_max = Duration::MAX, + "stale_after" => config.stale_after = Duration::MAX, + _ => unreachable!(), + } + assert!(matches!( + BootstrapGossipDiscovery::new(config, BootstrapClientConfig::new(NodeId::new("client"))), + Err(DiscoveryError::InvalidConfiguration { provider, message }) + if provider == PROVIDER && message.contains(field) + )); + } + } + + #[test] + fn runtime_overflow_disables_retry_even_after_announcement() { + let now = Instant::now(); + let mut config = BootstrapGossipDiscoveryConfig::new(vec!["seed:9000".into()]); + let mut schedule = SeedSchedule { + next_probe: now, + not_before: now, + retry_delay: Duration::MAX, + disabled: false, + }; + let error = BootstrapError::from(NetError::Wire(nx_net::WireError::RateLimited { + retry_after_ms: Some(200), + })); + assert!(matches!( + schedule.failed(&config, &error, now), + Err(DiscoveryError::Provider { + retryable: false, + .. + }) + )); + schedule.announce(now); + assert_eq!(schedule.deadline(), None); + assert!(schedule.not_before >= now + Duration::from_millis(200)); + + config.refresh_interval = Duration::MAX; + assert!(seed_deadlines(now, &config, Duration::from_secs(1)).is_err()); + config.refresh_interval = Duration::from_secs(1); + config.stale_after = Duration::MAX; + assert!(seed_deadlines(now, &config, Duration::MAX).is_err()); + } + + #[test] + fn representable_seed_policy_rejects_overflow_after_clock_advance() { + let config = BootstrapGossipDiscoveryConfig::new(vec!["seed:9000".into()]); + validate_config(&config).unwrap(); + let now = super::super::dynamic::deadline_boundary(); + let mut schedule = SeedSchedule { + next_probe: now, + not_before: now, + retry_delay: config.retry_initial, + disabled: false, + }; + let error = BootstrapError::from(NetError::Wire(nx_net::WireError::RateLimited { + retry_after_ms: Some(2000), + })); + assert!(schedule.failed(&config, &error, now).is_err()); + schedule.announce(now); + assert_eq!(schedule.deadline(), None); + assert!(seed_deadlines(now, &config, Duration::from_secs(1)).is_err()); + } + + async fn assert_panicked_shutdown_withdraws(restart: bool) { + let seed = Node::try_new_with_bootstrap_server( + NodeConfig::new(NodeId::new("seed"), "127.0.0.1:0"), + BootstrapServerConfig::new("default").unwrap(), + ) + .unwrap(); + let bound = seed.start_listener().await.unwrap().to_string(); + seed.announce_bootstrap_endpoint(bound.clone()).unwrap(); + let mut config = BootstrapGossipDiscoveryConfig::new(vec![bound.clone()]); + config.max_candidates = 4; + config.refresh_interval = Duration::from_millis(10); + let provider = BootstrapGossipDiscovery::new( + config, + BootstrapClientConfig::new(NodeId::new("client")), + ) + .unwrap(); + let advertised = "127.0.0.1:43111"; + provider + .announce(&PeerAnnouncement { + endpoint: advertised.into(), + }) + .await + .unwrap(); + let mut events = provider.watch().await.unwrap(); + tokio::time::timeout(Duration::from_secs(2), events.recv()) + .await + .unwrap() + .unwrap(); + let observer = + BootstrapClient::new(BootstrapClientConfig::new(NodeId::new("observer"))).unwrap(); + let before = observer + .query(&bound, BootstrapRequest::new("default", 4)) + .await + .unwrap(); + assert!(before.endpoints.contains(&advertised.to_string())); + + let old = provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .clone() + .unwrap(); + provider.inner.state.panic_on_next_observation(); + super::super::dynamic::assert_invalidated(&mut events).await; + let result = tokio::time::timeout(Duration::from_secs(5), old.clone().join()) + .await + .unwrap(); + assert!( + matches!(result, Err(DiscoveryError::Provider { retryable: false, message, .. }) + if message.contains("provider task failed") && message.contains("panic")) + ); + assert!(provider.inner.state.snapshot().peers().is_empty()); + assert!(provider.inner.state.watch().snapshot().peers().is_empty()); + assert!(provider.inner.announced_seeds.lock().unwrap().is_empty()); + let after = observer + .query(&bound, BootstrapRequest::new("default", 4)) + .await + .unwrap(); + assert_eq!(after.endpoints, std::slice::from_ref(&bound)); + if restart { + let (first, second) = tokio::join!(provider.watch(), provider.watch()); + let mut first = first.unwrap(); + second.unwrap(); + let current = provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .clone() + .unwrap(); + assert!(!old.same_generation(¤t)); + provider.watch().await.unwrap(); + assert!( + current.same_generation( + provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .as_ref() + .unwrap() + ) + ); + tokio::time::timeout(Duration::from_secs(2), first.recv()) + .await + .unwrap() + .unwrap(); + let renewed = observer + .query(&bound, BootstrapRequest::new("default", 4)) + .await + .unwrap(); + assert!(renewed.endpoints.contains(&advertised.to_string())); + provider.shutdown().await.unwrap(); + } else { + assert!(provider.shutdown().await.is_err()); + } + assert!(provider.watch().await.is_err()); + assert!(provider.inner.announcement_tx.borrow().is_none()); + seed.shutdown().await; + } + + #[tokio::test] + async fn panicked_probe_still_withdraws_and_clears_snapshot() { + assert_panicked_shutdown_withdraws(false).await; + } + + #[tokio::test] + async fn panic_invalidates_and_concurrent_subscribers_restart_one_bootstrap_generation() { + assert_panicked_shutdown_withdraws(true).await; + } + + #[derive(Clone, Copy)] + enum AnnouncementAck { + Pending, + Panic, + Lost, + } + + #[derive(Clone)] + struct AppliedWithoutAck { + applied: Arc>>, + calls: tokio::sync::mpsc::Sender<&'static str>, + withdrawal_ack: Arc>>>, + announcement_ack: AnnouncementAck, + } + + #[async_trait] + impl SeedClient for AppliedWithoutAck { + async fn query( + &self, + _seed: &str, + request: BootstrapRequest, + ) -> Result { + if let Some(endpoint) = request.advertised_endpoint { + *self.applied.lock().unwrap() = Some(endpoint); + self.calls.send("applied-without-ack").await.unwrap(); + return match self.announcement_ack { + AnnouncementAck::Pending => pending().await, + AnnouncementAck::Panic => panic!("injected probe panic after seed application"), + AnnouncementAck::Lost => Err(NetError::Timeout.into()), + }; + } + self.applied.lock().unwrap().take(); + self.calls.send("withdrawal-applied").await.unwrap(); + if let Some(ack) = self.withdrawal_ack.lock().await.take() { + ack.await.unwrap(); + } + Ok(nx_net::BootstrapResponse { + seed_node_id: NodeId::new("seed"), + endpoints: Vec::new(), + candidate_ttl: Duration::from_secs(30), + }) + } + } + + async fn assert_lost_ack_is_withdrawn(announcement_ack: AnnouncementAck, drop_provider: bool) { + let panic = matches!(announcement_ack, AnnouncementAck::Panic); + let mut config = BootstrapGossipDiscoveryConfig::new(vec!["seed:9000".into()]); + config.max_candidates = 4; + let provider = BootstrapGossipDiscovery::new( + config.clone(), + BootstrapClientConfig::new(NodeId::new("client")), + ) + .unwrap(); + provider + .announce(&PeerAnnouncement { + endpoint: "local:9000".into(), + }) + .await + .unwrap(); + let (calls, mut call_rx) = tokio::sync::mpsc::channel(8); + let (release, released) = tokio::sync::oneshot::channel(); + let client = AppliedWithoutAck { + applied: Arc::new(StdMutex::new(None)), + calls, + withdrawal_ack: Arc::new(tokio::sync::Mutex::new(Some(released))), + announcement_ack, + }; + let cleanup = BootstrapCleanup { + client: client.clone(), + cluster_id: config.cluster_id.clone(), + announcement_tx: provider.inner.announcement_tx.clone(), + announced_seeds: provider.inner.announced_seeds.clone(), + state: provider.inner.state.clone(), + preserve_announcement: false, + }; + let (stop, stop_rx) = watch::channel(false); + // A full prior view must be cleared even if query panics before any ACK. + provider.inner.state.observe(vec!["cached:9000".into()]); + let mut events = provider.inner.state.watch(); + let state = provider.inner.state.clone(); + let announcement = provider.inner.announcement_tx.subscribe(); + let seeds = provider.inner.announced_seeds.clone(); + let worker_client = client.clone(); + let task = ProviderTask::spawn( + PROVIDER, + state.clone(), + stop_rx.clone(), + async move { + run_bootstrap(config, worker_client, state, announcement, seeds, stop_rx).await; + Ok(()) + }, + move || async move { cleanup.withdraw().await }, + ); + let completion = task.clone(); + { + let mut lifecycle = provider.inner.lifecycle.lock().unwrap(); + lifecycle.shutdown = Some(stop); + lifecycle.task = Some(task); + } + tokio::time::timeout(Duration::from_secs(2), async { + assert_eq!(call_rx.recv().await, Some("applied-without-ack")); + assert!(provider.inner.announced_seeds.lock().unwrap().contains("seed:9000")); + if panic { + super::super::dynamic::assert_invalidated(&mut events).await; + } else { + assert_eq!(client.applied.lock().unwrap().as_deref(), Some("local:9000")); + } + if matches!(announcement_ack, AnnouncementAck::Lost) { + // Publication follows processing the failed query. The seed + // must remain tracked even after the timeout result is handled. + assert!(super::super::observed_peers(events.recv().await.unwrap().change).is_empty()); + assert!(provider.inner.announced_seeds.lock().unwrap().contains("seed:9000")); + } + if drop_provider { + let state = provider.inner.state.clone(); + let seeds = provider.inner.announced_seeds.clone(); + let announcement = provider.inner.announcement_tx.clone(); + drop(provider); + assert_eq!(call_rx.recv().await, Some("withdrawal-applied")); + assert!(!completion.completion_ready()); + release.send(()).unwrap(); + completion.join().await.unwrap(); + assert!(client.applied.lock().unwrap().is_none()); + assert!(state.snapshot().peers().is_empty()); + assert!(seeds.lock().unwrap().is_empty()); + assert!(announcement.borrow().is_none()); + return; + } + let mut waiter = Box::pin(provider.shutdown()); + std::future::poll_fn(|cx| { + assert!(waiter.as_mut().poll(cx).is_pending()); + std::task::Poll::Ready(()) + }).await; + assert_eq!(call_rx.recv().await, Some("withdrawal-applied")); + drop(waiter); + assert!(!completion.completion_ready()); + // The seed applied withdrawal; only its ACK is deliberately held. + assert!(client.applied.lock().unwrap().is_none()); + release.send(()).unwrap(); + let result = completion.join().await; + if panic { + assert!(matches!(result, Err(DiscoveryError::Provider { message, .. }) if message.contains("panic"))); + assert!(provider.shutdown().await.is_err()); + } else { + result.unwrap(); + provider.shutdown().await.unwrap(); + } + assert!(provider.inner.state.snapshot().peers().is_empty()); + assert!(provider.inner.announced_seeds.lock().unwrap().is_empty()); + assert!(provider.inner.announcement_tx.borrow().is_none()); + assert!(call_rx.try_recv().is_err()); + assert!(provider.watch().await.is_err()); + }).await.unwrap(); + } + + #[tokio::test] + async fn lost_announcement_ack_is_withdrawn_after_cancelled_shutdown_wait() { + assert_lost_ack_is_withdrawn(AnnouncementAck::Pending, false).await; + } + + #[tokio::test] + async fn timed_out_announcement_ack_keeps_seed_tracked_for_withdrawal() { + assert_lost_ack_is_withdrawn(AnnouncementAck::Lost, false).await; + } + + #[tokio::test] + async fn panic_after_seed_application_still_withdraws_without_announcement_ack() { + assert_lost_ack_is_withdrawn(AnnouncementAck::Panic, false).await; + } + + #[tokio::test] + async fn dropping_provider_still_withdraws_an_announcement_without_ack() { + assert_lost_ack_is_withdrawn(AnnouncementAck::Pending, true).await; + } + + #[tokio::test] + async fn missing_withdrawal_ack_remains_bounded_best_effort_and_idempotent() { + let (calls, mut call_rx) = tokio::sync::mpsc::channel(8); + let (_release, released) = tokio::sync::oneshot::channel(); + let client = AppliedWithoutAck { + applied: Arc::new(StdMutex::new(Some("local:9000".into()))), + calls, + withdrawal_ack: Arc::new(tokio::sync::Mutex::new(Some(released))), + announcement_ack: AnnouncementAck::Pending, + }; + let (announcement_tx, _) = watch::channel(Some("local:9000".into())); + let cleanup = BootstrapCleanup { + client: client.clone(), + cluster_id: "default".into(), + announcement_tx, + announced_seeds: Arc::new(StdMutex::new(HashSet::from(["seed:9000".into()]))), + state: Arc::new(DynamicState::new(8)), + preserve_announcement: false, + }; + tokio::time::timeout( + SHUTDOWN_WITHDRAWAL_BUDGET + Duration::from_secs(1), + cleanup.withdraw(), + ) + .await + .unwrap() + .unwrap(); + assert_eq!(call_rx.recv().await, Some("withdrawal-applied")); + assert!(client.applied.lock().unwrap().is_none()); + // Repeating removal of the same NodeId remains harmless. + cleanup.withdraw().await.unwrap(); + assert_eq!(call_rx.recv().await, Some("withdrawal-applied")); + assert!(client.applied.lock().unwrap().is_none()); + } + + struct ControlledClient { + calls: tokio::sync::mpsc::Sender<(String, Instant)>, + limited_calls: std::sync::atomic::AtomicUsize, + } + + #[async_trait] + impl SeedClient for ControlledClient { + async fn query( + &self, + seed: &str, + _request: BootstrapRequest, + ) -> Result { + self.calls + .send((seed.to_string(), Instant::now())) + .await + .unwrap(); + if seed == "limited:1" + && self + .limited_calls + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + > 0 + { + return Err(NetError::Wire(nx_net::WireError::RateLimited { + retry_after_ms: Some(200), + }) + .into()); + } + Ok(nx_net::BootstrapResponse { + seed_node_id: NodeId::new(seed), + endpoints: Vec::new(), + candidate_ttl: Duration::from_millis(40), + }) + } + } + + #[tokio::test] + async fn retry_after_survives_actual_view_expiry_and_announcement_while_healthy_seeds_progress() + { + let mut config = + BootstrapGossipDiscoveryConfig::new(vec!["limited:1".into(), "healthy:2".into()]); + config.refresh_interval = Duration::from_millis(10); + config.retry_initial = Duration::from_millis(10); + config.retry_max = Duration::from_millis(100); // configured cap still applies + let state = Arc::new(DynamicState::new(128)); + let mut events = state.watch(); + let (calls, mut calls_rx) = tokio::sync::mpsc::channel(128); + let client = ControlledClient { + calls, + limited_calls: std::sync::atomic::AtomicUsize::new(0), + }; + let (announcement, announcement_rx) = watch::channel(None); + let (shutdown, shutdown_rx) = watch::channel(false); + let task = tokio::spawn(run_bootstrap( + config, + client, + state, + announcement_rx, + Arc::new(StdMutex::new(HashSet::new())), + shutdown_rx, + )); + tokio::time::timeout(Duration::from_secs(2), async { + let mut limited = 0; + let limited_at = loop { + let (seed, at) = calls_rx.recv().await.unwrap(); + if seed == "limited:1" { + limited += 1; + } + if limited == 2 { + break at; + } + }; + // Wait for the limited seed's retained view to actually disappear. + loop { + let peers = super::super::observed_peers(events.recv().await.unwrap().change); + if !peers.contains(&"limited:1".to_string()) { + break; + } + } + announcement.send_replace(Some("local:3".into())); + let mut healthy_progress = false; + loop { + let (seed, at) = calls_rx.recv().await.unwrap(); + if seed == "healthy:2" + && at >= limited_at + && at < limited_at + Duration::from_millis(100) + { + healthy_progress = true; + } + if seed == "limited:1" { + assert!(at >= limited_at + Duration::from_millis(100)); + assert!(healthy_progress); + break; + } + } + }) + .await + .unwrap(); + shutdown.send_replace(true); + task.await.unwrap(); + } + + #[test] + fn cached_seed_views_do_not_refresh_observations_on_failure_or_other_seed_expiry() { + let config = BootstrapGossipDiscoveryConfig::new(vec!["a:1".into(), "b:2".into()]); + let state = DynamicState::new(8); + let now = std::time::Instant::now(); + let mut views = HashMap::from([ + ( + "a:1".into(), + SeedView { + endpoints: vec!["a:1".into()], + expires_at: Instant::now() + Duration::from_secs(10), + observed_at: now, + }, + ), + ( + "b:2".into(), + SeedView { + endpoints: vec!["b:2".into()], + expires_at: Instant::now() + Duration::from_secs(10), + observed_at: now, + }, + ), + ]); + publish_views(&config, &mut views, &state); + let first = state.snapshot(); + publish_views(&config, &mut views, &state); + assert_eq!(first, state.snapshot()); + views.get_mut("b:2").unwrap().expires_at = Instant::now(); + publish_views(&config, &mut views, &state); + assert_eq!(state.snapshot().peers(), ["a:1"]); + assert_eq!(state.snapshot().observations().unwrap(), [now]); + } + + #[test] + fn views_are_bounded_deduplicated_and_follow_seed_order() { + let views = HashMap::from([ + ( + "a:1".into(), + SeedView { + endpoints: vec!["a:1".into(), "shared:3".into()], + expires_at: Instant::now() + Duration::from_secs(1), + observed_at: std::time::Instant::now(), + }, + ), + ( + "b:2".into(), + SeedView { + endpoints: vec!["b:2".into(), "shared:3".into()], + expires_at: Instant::now() + Duration::from_secs(1), + observed_at: std::time::Instant::now(), + }, + ), + ]); + assert_eq!( + flatten_views(&["b:2".into(), "a:1".into()], &views, 3), + ["b:2", "shared:3", "a:1"] + ); + } + + #[test] + fn invalid_or_unbounded_seed_configuration_is_rejected() { + let mut config = BootstrapGossipDiscoveryConfig::new(vec!["seed:9000".into()]); + config.max_seeds = 0; + assert!(validate_config(&config).is_err()); + + config.max_seeds = 1; + config.max_candidates = nx_net::MAX_BOOTSTRAP_RESPONSE_CAPACITY; + assert!(validate_config(&config).is_ok()); + config.max_candidates += 1; + assert!(validate_config(&config).is_err()); + } + + #[test] + fn expiry_and_announcement_do_not_override_a_seeds_not_before() { + let now = Instant::now(); + let mut schedule = SeedSchedule { + next_probe: now, + not_before: now + Duration::from_secs(30), + retry_delay: Duration::from_secs(1), + disabled: false, + }; + let mut healthy = SeedSchedule { + next_probe: now + Duration::from_secs(5), + not_before: now, + retry_delay: Duration::from_secs(1), + disabled: false, + }; + schedule.announce(now + Duration::from_secs(2)); + healthy.announce(now + Duration::from_secs(2)); + assert_eq!(schedule.deadline(), Some(now + Duration::from_secs(30))); + assert_eq!(healthy.deadline(), Some(now + Duration::from_secs(2))); + // An expiry wakeup never changes the per-seed schedule. + let expiry = now + Duration::from_secs(3); + assert!(schedule.deadline().unwrap() > expiry); + } + + #[tokio::test] + async fn candidate_view_expires_while_a_seed_query_is_stalled() { + let config = BootstrapGossipDiscoveryConfig::new(vec!["seed:9000".into()]); + let state = DynamicState::new(8); + state.replace(vec!["peer:9000".into()]); + let mut watch = state.watch(); + let mut views = HashMap::from([( + "seed:9000".into(), + SeedView { + endpoints: vec!["peer:9000".into()], + expires_at: Instant::now() + Duration::from_millis(10), + observed_at: std::time::Instant::now(), + }, + )]); + let (_shutdown_tx, mut shutdown_rx) = watch::channel(false); + + let wait = await_query_with_expiry( + std::future::pending::<()>(), + &config, + &mut views, + &state, + &mut shutdown_rx, + ); + tokio::pin!(wait); + let event = tokio::time::timeout(Duration::from_secs(1), async { + tokio::select! { + _ = &mut wait => panic!("pending query unexpectedly completed"), + event = watch.recv() => event.unwrap(), + } + }) + .await + .unwrap(); + + assert_eq!( + super::super::observed_peers(event.change), + Vec::::new() + ); + } + + #[tokio::test] + async fn provider_learns_candidates_and_withdraws_its_announcement() { + let seed = Node::try_new_with_bootstrap_server( + NodeConfig::new(NodeId::new("seed"), "127.0.0.1:0"), + BootstrapServerConfig::new("cluster-a") + .unwrap() + .with_max_response_candidates(4) + .unwrap(), + ) + .unwrap(); + let bound = seed.start_listener().await.unwrap(); + seed.announce_bootstrap_endpoint(bound.to_string()).unwrap(); + + let mut client_config = BootstrapClientConfig::new(NodeId::new("client")); + client_config.max_response_candidates = 4; + let mut config = BootstrapGossipDiscoveryConfig::new(vec![bound.to_string()]); + config.cluster_id = "cluster-a".into(); + config.max_candidates = 4; + config.refresh_interval = Duration::from_secs(1); + config.retry_initial = Duration::from_millis(10); + config.retry_max = Duration::from_millis(20); + let provider = BootstrapGossipDiscovery::new(config, client_config).unwrap(); + provider + .announce(&PeerAnnouncement { + endpoint: "127.0.0.1:43111".into(), + }) + .await + .unwrap(); + let mut watch = provider.watch().await.unwrap(); + + let event = tokio::time::timeout(Duration::from_secs(2), watch.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!( + super::super::observed_peers(event.change), + vec![bound.to_string()] + ); + provider.shutdown().await.unwrap(); + provider.shutdown().await.unwrap(); + assert!(provider.inner.state.snapshot().peers().is_empty()); + assert!(provider.inner.announced_seeds.lock().unwrap().is_empty()); + assert!(provider.inner.announcement_tx.borrow().is_none()); + assert!(provider.watch().await.is_err()); + + let observer = + BootstrapClient::new(BootstrapClientConfig::new(NodeId::new("observer"))).unwrap(); + let response = observer + .query(&bound.to_string(), BootstrapRequest::new("cluster-a", 4)) + .await + .unwrap(); + assert_eq!(response.endpoints, [bound.to_string()]); + seed.shutdown().await; + } + + #[tokio::test] + async fn provider_expires_candidates_and_recovers_after_seed_restart() { + let server_config = BootstrapServerConfig::new("cluster-a") + .unwrap() + .with_candidate_ttl(Duration::from_millis(40)) + .unwrap(); + let seed = Node::try_new_with_bootstrap_server( + NodeConfig::new(NodeId::new("seed"), "127.0.0.1:0"), + server_config.clone(), + ) + .unwrap(); + let bound = seed.start_listener().await.unwrap(); + seed.announce_bootstrap_endpoint(bound.to_string()).unwrap(); + + let mut client_config = BootstrapClientConfig::new(NodeId::new("client")); + client_config.max_response_candidates = 4; + let mut config = BootstrapGossipDiscoveryConfig::new(vec![bound.to_string()]); + config.cluster_id = "cluster-a".into(); + config.max_candidates = 4; + config.refresh_interval = Duration::from_millis(10); + config.retry_initial = Duration::from_millis(10); + config.retry_max = Duration::from_millis(20); + config.stale_after = Duration::from_millis(40); + let provider = BootstrapGossipDiscovery::new(config, client_config).unwrap(); + let mut watch = provider.watch().await.unwrap(); + + let discovered = tokio::time::timeout(Duration::from_secs(2), watch.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!( + super::super::observed_peers(discovered.change), + vec![bound.to_string()] + ); + + seed.shutdown().await; + assert!( + super::super::next_changed_peers(&mut watch, &[bound.to_string()]) + .await + .is_empty() + ); + + let restarted = Node::try_new_with_bootstrap_server( + NodeConfig::new(NodeId::new("seed-restarted"), bound.to_string()), + server_config, + ) + .unwrap(); + restarted.start_listener().await.unwrap(); + restarted + .announce_bootstrap_endpoint(bound.to_string()) + .unwrap(); + let recovered = tokio::time::timeout(Duration::from_secs(2), watch.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!( + super::super::observed_peers(recovered.change), + vec![bound.to_string()] + ); + + provider.shutdown().await.unwrap(); + restarted.shutdown().await; + } +} diff --git a/crates/nx-core/src/discovery/dns_srv.rs b/crates/nx-core/src/discovery/dns_srv.rs new file mode 100644 index 0000000..6bec9b0 --- /dev/null +++ b/crates/nx-core/src/discovery/dns_srv.rs @@ -0,0 +1,980 @@ +use std::sync::{Arc, Mutex as StdMutex}; +use std::time::Duration; + +use async_trait::async_trait; +use hickory_resolver::TokioResolver; +use hickory_resolver::net::{DnsError, NetError as DnsNetError}; +use hickory_resolver::proto::rr::rdata::SRV; +use hickory_resolver::proto::rr::{RData, RecordType}; +use tokio::sync::watch; +use tokio::time::Instant; + +use super::dynamic::{DynamicState, ProviderTask, checked_deadline, validate_durations}; +use super::{ + DEFAULT_DISCOVERY_CLUSTER, DEFAULT_DISCOVERY_EVENT_CAPACITY, DEFAULT_MAX_PEER_CANDIDATES, + DiscoveryError, DiscoverySnapshot, DiscoveryWatch, PeerAnnouncement, PeerDiscovery, + validate_event_capacity, +}; + +const PROVIDER: &str = "dns-srv"; +const DEFAULT_RETRY_INTERVAL: Duration = Duration::from_secs(5); +const DEFAULT_MAX_REFRESH_INTERVAL: Duration = Duration::from_secs(300); + +/// DNS-SRV lookup and refresh policy. +#[derive(Debug, Clone)] +pub struct DnsSrvDiscoveryConfig { + pub service_name: String, + pub cluster_id: String, + pub retry_interval: Duration, + pub max_refresh_interval: Duration, + pub max_candidates: usize, + /// Event channel capacity in `1..=super::MAX_DISCOVERY_EVENT_CAPACITY`. + /// Defaults to [`DEFAULT_DISCOVERY_EVENT_CAPACITY`]; validated by the provider constructor. + pub event_capacity: usize, +} + +impl DnsSrvDiscoveryConfig { + pub fn new(service_name: impl Into) -> Self { + Self { + service_name: service_name.into(), + cluster_id: DEFAULT_DISCOVERY_CLUSTER.to_string(), + retry_interval: DEFAULT_RETRY_INTERVAL, + max_refresh_interval: DEFAULT_MAX_REFRESH_INTERVAL, + max_candidates: DEFAULT_MAX_PEER_CANDIDATES, + event_capacity: DEFAULT_DISCOVERY_EVENT_CAPACITY, + } + } +} + +#[derive(Debug)] +struct SrvAnswer { + records: Vec, + valid_until: Instant, +} + +#[async_trait] +trait SrvResolver: Send + Sync { + async fn lookup(&self, config: &DnsSrvDiscoveryConfig) -> Result; +} + +struct HickorySrvResolver(TokioResolver); + +#[async_trait] +impl SrvResolver for HickorySrvResolver { + async fn lookup(&self, config: &DnsSrvDiscoveryConfig) -> Result { + lookup(config, &self.0).await + } +} + +struct Lifecycle { + stopped: bool, + shutdown: Option>, + task: Option, +} + +struct Inner { + config: DnsSrvDiscoveryConfig, + state: Arc, + resolver: Arc, + lifecycle: StdMutex, +} + +impl Drop for Inner { + fn drop(&mut self) { + let lifecycle = self + .lifecycle + .get_mut() + .unwrap_or_else(|error| error.into_inner()); + if let Some(shutdown) = lifecycle.shutdown.take() { + let _ = shutdown.send(true); + } + } +} + +/// Discovers connection candidates from a DNS SRV record. +/// +/// SRV priority is retained as deterministic ordering metadata only; results +/// remain unauthenticated candidates. A successful empty/NXDOMAIN response +/// removes the previous view. Transient resolver failures keep the last valid +/// view only until its DNS expiry. +pub struct DnsSrvDiscovery { + inner: Arc, +} + +impl DnsSrvDiscovery { + pub fn new(config: DnsSrvDiscoveryConfig) -> Result { + validate_config(&config)?; + let resolver = TokioResolver::builder_tokio() + .and_then(|builder| builder.build()) + .map_err(|error| { + provider_error( + format!("cannot load system DNS configuration: {error}"), + false, + ) + })?; + Ok(Self::with_resolver( + config, + Arc::new(HickorySrvResolver(resolver)), + )) + } + + fn with_resolver(config: DnsSrvDiscoveryConfig, resolver: Arc) -> Self { + Self { + inner: Arc::new(Inner { + state: Arc::new(DynamicState::new(config.event_capacity)), + config, + resolver, + lifecycle: StdMutex::new(Lifecycle { + stopped: false, + shutdown: None, + task: None, + }), + }), + } + } + + async fn ensure_started(&self) -> Result<(), DiscoveryError> { + let mut lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + if lifecycle.stopped { + return Err(provider_error("provider is shut down", false)); + } + if let Some(task) = lifecycle.task.as_ref() + && task.running(PROVIDER, &self.inner.state)? + { + return Ok(()); + } + + let (shutdown, shutdown_rx) = watch::channel(false); + let config = self.inner.config.clone(); + let resolver = self.inner.resolver.clone(); + let state = Arc::clone(&self.inner.state); + lifecycle.shutdown = Some(shutdown); + lifecycle.task = Some(ProviderTask::spawn( + PROVIDER, + state.clone(), + shutdown_rx.clone(), + run_dns_refresh(config, resolver, state, shutdown_rx), + || async { Ok(()) }, + )); + Ok(()) + } +} + +#[async_trait] +impl PeerDiscovery for DnsSrvDiscovery { + fn cluster_id(&self) -> &str { + &self.inner.config.cluster_id + } + + async fn discover(&self) -> Result { + self.ensure_started().await?; + Ok(self.inner.state.snapshot()) + } + + async fn announce(&self, _announcement: &PeerAnnouncement) -> Result<(), DiscoveryError> { + Err(DiscoveryError::Unsupported { + provider: PROVIDER.to_string(), + operation: "announcement", + }) + } + + async fn watch(&self) -> Result { + self.ensure_started().await?; + self.inner.state.live_watch() + } + + fn request_shutdown(&self) { + let mut lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + lifecycle.stopped = true; + if let Some(shutdown) = lifecycle.shutdown.as_ref() { + let _ = shutdown.send(true); + } + } + + async fn shutdown(&self) -> Result<(), DiscoveryError> { + self.request_shutdown(); + let task = { + let lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + lifecycle.task.clone() + }; + match task { + Some(task) => task.join().await, + None => Ok(()), + } + } +} + +async fn run_dns_refresh( + config: DnsSrvDiscoveryConfig, + resolver: Arc, + state: Arc, + mut shutdown: watch::Receiver, +) -> Result<(), DiscoveryError> { + let mut valid_until = None; + let mut next_refresh = Instant::now(); + loop { + if *shutdown.borrow() || shutdown.has_changed().is_err() { + return Ok(()); + } + tokio::select! { + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { break; } + } + _ = tokio::time::sleep_until(next_refresh) => { + let query = resolver.lookup(&config); + tokio::pin!(query); + let result = loop { + tokio::select! { + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return Ok(()); + } + } + result = &mut query => break result, + _ = wait_for_dns_expiry(valid_until) => { + state.replace(Vec::new()); + valid_until = None; + } + } + }; + match result { + Ok(answer) => { + (valid_until, next_refresh) = apply_dns_answer(&config, &state, answer, valid_until)?; + } + Err(error) => { + let now = Instant::now(); + if valid_until.is_some_and(|deadline| now >= deadline) { + state.replace(Vec::new()); + } + tracing::warn!(%error, name = %config.service_name, "DNS-SRV discovery refresh failed"); + if matches!(error, DiscoveryError::Provider { retryable: false, .. }) { + return Err(error); + } + next_refresh = retry_deadline(now, config.retry_interval, valid_until)?; + } + } + } + } + } + Ok(()) +} + +fn apply_dns_answer( + config: &DnsSrvDiscoveryConfig, + state: &DynamicState, + answer: SrvAnswer, + previous_valid_until: Option, +) -> Result<(Option, Instant), DiscoveryError> { + let now = Instant::now(); + if answer.valid_until <= now { + state.replace(Vec::new()); + return Ok(( + None, + checked_deadline(now, config.retry_interval, PROVIDER, "retry_interval")?, + )); + } + let refresh = checked_deadline( + now, + config.max_refresh_interval, + PROVIDER, + "max_refresh_interval", + )?; + let peers = records_to_peers(answer.records, config.max_candidates); + if previous_valid_until.is_some_and(|previous| answer.valid_until <= previous) { + // Hickory can return the same cached answer before its original expiry. + // Only a newly validated DNS lifetime renews candidate observations. + state.replace(peers); + } else { + state.observe(peers); + } + Ok((Some(answer.valid_until), answer.valid_until.min(refresh))) +} + +async fn wait_for_dns_expiry(deadline: Option) { + match deadline { + Some(deadline) => tokio::time::sleep_until(deadline).await, + None => std::future::pending().await, + } +} + +async fn lookup( + config: &DnsSrvDiscoveryConfig, + resolver: &TokioResolver, +) -> Result { + match inner_lookup(config, resolver).await { + Ok(answer) => Ok(answer), + Err(DnsNetError::Dns(DnsError::NoRecordsFound(no_records))) => { + let negative_ttl = + bounded_negative_ttl(no_records.negative_ttl, config.max_refresh_interval); + Ok(SrvAnswer { + records: Vec::new(), + valid_until: checked_deadline( + Instant::now(), + negative_ttl, + PROVIDER, + "negative_ttl", + )?, + }) + } + Err(error) => Err(provider_error( + format!("lookup of {} failed: {error}", config.service_name), + true, + )), + } +} + +fn retry_deadline( + now: Instant, + retry_interval: Duration, + valid_until: Option, +) -> Result { + let retry = checked_deadline(now, retry_interval, PROVIDER, "retry_interval")?; + Ok(valid_until + .filter(|deadline| *deadline > now) + .map(|deadline| deadline.min(retry)) + .unwrap_or(retry)) +} + +fn bounded_negative_ttl(negative_ttl: Option, max_refresh_interval: Duration) -> Duration { + negative_ttl + .map(|seconds| Duration::from_secs(u64::from(seconds))) + .unwrap_or(max_refresh_interval) + .min(max_refresh_interval) +} + +async fn inner_lookup( + config: &DnsSrvDiscoveryConfig, + resolver: &TokioResolver, +) -> Result { + let lookup = resolver + .lookup(&config.service_name, RecordType::SRV) + .await?; + let valid_until = lookup.valid_until().into(); + let records = lookup + .answers() + .iter() + .filter_map(|record| match &record.data { + RData::SRV(srv) => Some(srv.clone()), + _ => None, + }) + .collect(); + Ok(SrvAnswer { + records, + valid_until, + }) +} + +fn records_to_peers(mut records: Vec, max_candidates: usize) -> Vec { + records.sort_by_key(|record| { + ( + record.priority, + record.target.to_utf8(), + record.port, + record.weight, + ) + }); + records + .into_iter() + .filter(|record| record.port != 0 && !record.target.is_root()) + .filter_map(|record| { + let endpoint = format!( + "{}:{}", + record.target.to_utf8().trim_end_matches('.'), + record.port + ); + crate::sync_manager::canonicalize_endpoint(&endpoint).ok() + }) + .fold(Vec::new(), |mut peers, endpoint| { + if peers.len() < max_candidates && !peers.contains(&endpoint) { + peers.push(endpoint); + } + peers + }) +} + +pub(super) fn validate_config(config: &DnsSrvDiscoveryConfig) -> Result<(), DiscoveryError> { + validate_event_capacity(PROVIDER, config.event_capacity)?; + if !config.service_name.ends_with('.') { + return Err(invalid( + "service_name must be a fully-qualified name ending with '.'", + )); + } + let labels = config + .service_name + .trim_end_matches('.') + .split('.') + .collect::>(); + let valid_service = labels.first().is_some_and(|label| { + label.len() > 1 + && label.starts_with('_') + && label[1..] + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + }); + let valid_protocol = labels.get(1).is_some_and(|label| { + label.eq_ignore_ascii_case("_tcp") || label.eq_ignore_ascii_case("_udp") + }); + if labels.len() < 3 || !valid_service || !valid_protocol { + return Err(invalid( + "service_name must use the fully-qualified _service._tcp|_udp.domain. form", + )); + } + if hickory_resolver::proto::rr::Name::from_ascii(&config.service_name).is_err() { + return Err(invalid("service_name is not a valid DNS name")); + } + if config.cluster_id.trim().is_empty() { + return Err(invalid("cluster_id must not be empty")); + } + if config.retry_interval.is_zero() + || config.max_refresh_interval.is_zero() + || config.max_candidates == 0 + { + return Err(invalid("intervals and limits must be greater than zero")); + } + validate_durations( + PROVIDER, + &[ + ("retry_interval", config.retry_interval), + ("max_refresh_interval", config.max_refresh_interval), + ], + ) +} + +fn invalid(message: impl Into) -> DiscoveryError { + DiscoveryError::InvalidConfiguration { + provider: PROVIDER.to_string(), + message: message.into(), + } +} + +fn provider_error(message: impl Into, retryable: bool) -> DiscoveryError { + DiscoveryError::Provider { + provider: PROVIDER.to_string(), + message: message.into(), + retryable, + } +} + +#[cfg(test)] +mod tests { + use std::collections::VecDeque; + use std::sync::Mutex; + + use hickory_resolver::proto::rr::Name; + + use super::*; + + #[test] + fn extreme_durations_are_rejected_before_resolver_construction() { + for field in ["retry_interval", "max_refresh_interval"] { + let mut config = DnsSrvDiscoveryConfig::new("_numax._tcp.example."); + if field == "retry_interval" { + config.retry_interval = Duration::MAX; + } else { + config.max_refresh_interval = Duration::MAX; + } + assert!(matches!(DnsSrvDiscovery::new(config), + Err(DiscoveryError::InvalidConfiguration { provider, message }) + if provider == PROVIDER && message.contains(field))); + } + } + + #[test] + fn runtime_deadline_overflow_does_not_publish_or_renew_cached_answers() { + let mut config = DnsSrvDiscoveryConfig::new("_numax._tcp.example."); + let now = Instant::now(); + let state = DynamicState::new(8); + state.observe_at(vec![("peer.example:9000".into(), now.into_std())]); + let cached = state.snapshot(); + let valid_until = now + Duration::from_secs(10); + config.max_refresh_interval = Duration::MAX; + let error = apply_dns_answer( + &config, + &state, + SrvAnswer { + records: vec![SRV::new( + 0, + 0, + 9000, + Name::from_ascii("peer.example.").unwrap(), + )], + valid_until, + }, + Some(valid_until), + ) + .unwrap_err(); + assert!(matches!( + error, + DiscoveryError::Provider { + retryable: false, + .. + } + )); + assert_eq!(state.snapshot(), cached); + assert!(retry_deadline(now, Duration::MAX, Some(valid_until)).is_err()); + assert!(retry_deadline(now, Duration::MAX, None).is_err()); + let boundary = super::super::dynamic::deadline_boundary(); + assert!(retry_deadline(boundary, config.retry_interval, None).is_err()); + assert!( + checked_deadline( + now, + bounded_negative_ttl(None, Duration::MAX), + PROVIDER, + "negative_ttl" + ) + .is_err() + ); + config.retry_interval = Duration::MAX; + assert!( + apply_dns_answer( + &config, + &state, + SrvAnswer { + records: Vec::new(), + valid_until: now, + }, + Some(valid_until) + ) + .is_err() + ); + assert!(state.snapshot().peers().is_empty()); + } + + #[tokio::test] + async fn panicked_refresh_is_reported_after_clearing_snapshot() { + let (provider, mut events) = panic_provider().await; + super::super::dynamic::assert_invalidated(&mut events).await; + let result = provider.shutdown().await; + assert!( + matches!(result, Err(DiscoveryError::Provider { retryable: false, message, .. }) + if message.contains("provider task failed") && message.contains("panic")) + ); + assert!(provider.inner.state.snapshot().peers().is_empty()); + assert!(provider.inner.state.watch().snapshot().peers().is_empty()); + assert!(provider.watch().await.is_err()); + } + + async fn panic_provider() -> (DnsSrvDiscovery, DiscoveryWatch) { + let record = SRV::new(0, 0, 9000, Name::from_ascii("cached.example.").unwrap()); + let resolver = Arc::new(SequenceResolver { + steps: Mutex::new(VecDeque::from([ + ResolverStep::Success(vec![record.clone()], Duration::from_secs(30)), + ResolverStep::Panic, + ResolverStep::Success(vec![record], Duration::from_secs(30)), + ])), + }); + let mut config = DnsSrvDiscoveryConfig::new("_numax._tcp.example."); + config.max_refresh_interval = Duration::from_millis(10); + let provider = DnsSrvDiscovery::with_resolver(config, resolver); + let mut events = provider.watch().await.unwrap(); + assert_eq!( + super::super::next_changed_peers(&mut events, &[]).await, + ["cached.example:9000"] + ); + (provider, events) + } + + #[tokio::test] + async fn panic_invalidates_and_concurrent_subscribers_restart_one_dns_generation() { + let (provider, mut events) = panic_provider().await; + let old = provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .clone() + .unwrap(); + super::super::dynamic::assert_invalidated(&mut events).await; + assert!(old.clone().join().await.is_err()); + assert!(provider.inner.state.snapshot().peers().is_empty()); + let (first, second) = tokio::join!(provider.watch(), provider.watch()); + let mut first = first.unwrap(); + second.unwrap(); + let current = provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .clone() + .unwrap(); + assert!(!old.same_generation(¤t)); + provider.watch().await.unwrap(); + assert!( + current.same_generation( + provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .as_ref() + .unwrap() + ) + ); + assert_eq!( + super::super::next_changed_peers(&mut first, &[]).await, + ["cached.example:9000"] + ); + provider.shutdown().await.unwrap(); + assert!(provider.watch().await.is_err()); + } + + #[tokio::test] + async fn fatal_refresh_error_invalidates_but_does_not_restart() { + let record = SRV::new(0, 0, 9000, Name::from_ascii("cached.example.").unwrap()); + let resolver = Arc::new(SequenceResolver { + steps: Mutex::new(VecDeque::from([ + ResolverStep::Success(vec![record], Duration::from_secs(30)), + ResolverStep::Fatal, + ResolverStep::Panic, + ])), + }); + let mut config = DnsSrvDiscoveryConfig::new("_numax._tcp.example."); + config.max_refresh_interval = Duration::from_millis(10); + let provider = DnsSrvDiscovery::with_resolver(config, resolver.clone()); + let mut events = provider.watch().await.unwrap(); + assert_eq!( + super::super::next_changed_peers(&mut events, &[]).await, + ["cached.example:9000"] + ); + super::super::dynamic::assert_invalidated(&mut events).await; + let task = provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .clone() + .unwrap(); + assert_eq!( + task.clone().join().await, + Err(provider_error("injected fatal resolver error", false)) + ); + assert!(matches!( + provider.watch().await, + Err(DiscoveryError::Provider { + retryable: false, + .. + }) + )); + assert!(provider.inner.state.snapshot().peers().is_empty()); + assert!( + task.same_generation( + provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .as_ref() + .unwrap() + ) + ); + assert_eq!(resolver.steps.lock().unwrap().len(), 1); + assert!(provider.shutdown().await.is_err()); + } + + enum ResolverStep { + Success(Vec, Duration), + TransientFailure, + Panic, + Fatal, + } + + struct SequenceResolver { + steps: Mutex>, + } + + #[async_trait] + impl SrvResolver for SequenceResolver { + async fn lookup( + &self, + _config: &DnsSrvDiscoveryConfig, + ) -> Result { + let step = self.steps.lock().unwrap().pop_front(); + match step { + Some(ResolverStep::Panic) => panic!("injected DNS refresh panic"), + Some(ResolverStep::Fatal) => { + Err(provider_error("injected fatal resolver error", false)) + } + Some(ResolverStep::Success(records, ttl)) => Ok(SrvAnswer { + records, + valid_until: Instant::now() + ttl, + }), + Some(ResolverStep::TransientFailure) => { + Err(provider_error("temporary resolver failure", true)) + } + None => std::future::pending().await, + } + } + } + + struct PendingResolver; + + #[async_trait] + impl SrvResolver for PendingResolver { + async fn lookup( + &self, + _config: &DnsSrvDiscoveryConfig, + ) -> Result { + std::future::pending().await + } + } + + #[test] + fn identical_fresh_dns_answer_renews_but_cached_answer_preserves_observation() { + let config = DnsSrvDiscoveryConfig::new("_numax._tcp.example."); + let state = DynamicState::new(8); + let old = std::time::Instant::now() - Duration::from_secs(1); + state.observe_at(vec![("peer.example:9000".into(), old)]); + let record = SRV::new(0, 0, 9000, Name::from_ascii("peer.example.").unwrap()); + let valid_until = Instant::now() + Duration::from_secs(10); + apply_dns_answer( + &config, + &state, + SrvAnswer { + records: vec![record.clone()], + valid_until, + }, + None, + ) + .unwrap(); + let fresh = state.snapshot(); + assert_eq!(fresh.peers(), ["peer.example:9000"]); + assert!(fresh.observations().unwrap()[0] > old); + apply_dns_answer( + &config, + &state, + SrvAnswer { + records: vec![record], + valid_until, + }, + Some(valid_until), + ) + .unwrap(); + assert_eq!(state.snapshot(), fresh); + } + + #[tokio::test] + async fn dns_view_expires_even_while_refresh_is_stalled() { + let resolver = Arc::new(SequenceResolver { + steps: Mutex::new(VecDeque::from([ResolverStep::Success( + vec![SRV::new( + 0, + 0, + 9000, + Name::from_ascii("peer.example.").unwrap(), + )], + Duration::from_millis(30), + )])), + }); + let mut config = DnsSrvDiscoveryConfig::new("_numax._tcp.example."); + config.max_refresh_interval = Duration::from_millis(5); + let provider = DnsSrvDiscovery::with_resolver(config, resolver); + let mut watch = provider.watch().await.unwrap(); + tokio::time::timeout(Duration::from_secs(1), async { + assert_eq!( + super::super::observed_peers(watch.recv().await.unwrap().change), + ["peer.example:9000"] + ); + assert!(super::super::observed_peers(watch.recv().await.unwrap().change).is_empty()); + }) + .await + .unwrap(); + provider.shutdown().await.unwrap(); + } + + #[test] + fn srv_records_are_bounded_deduplicated_and_deterministic() { + let records = vec![ + SRV::new(20, 0, 9002, Name::from_ascii("b.example.").unwrap()), + SRV::new(10, 1, 9001, Name::from_ascii("a.example.").unwrap()), + SRV::new(10, 1, 9001, Name::from_ascii("a.example.").unwrap()), + ]; + assert_eq!( + records_to_peers(records, 2), + ["a.example:9001", "b.example:9002"] + ); + } + + #[test] + fn srv_records_reject_undialable_targets_and_normalize_dns_names() { + let records = vec![ + SRV::new(1, 0, 9000, Name::from_ascii("0.0.0.0.").unwrap()), + SRV::new(1, 0, 9000, Name::from_ascii("BAD_NAME.").unwrap()), + SRV::new(1, 0, 9000, Name::from_ascii("Peer.Example.").unwrap()), + SRV::new(1, 0, 9000, Name::from_ascii("peer.example.").unwrap()), + SRV::new(1, 0, 0, Name::from_ascii("zero.example.").unwrap()), + SRV::new(1, 0, 9000, Name::root()), + ]; + + assert_eq!(records_to_peers(records, 8), ["peer.example:9000"]); + } + + #[test] + fn invalid_configuration_is_rejected_without_starting_a_task() { + let mut config = DnsSrvDiscoveryConfig::new("not-srv.example"); + config.max_candidates = 0; + assert!(DnsSrvDiscovery::new(config).is_err()); + + assert!(DnsSrvDiscovery::new(DnsSrvDiscoveryConfig::new("_numax.example.")).is_err()); + assert!(DnsSrvDiscovery::new(DnsSrvDiscoveryConfig::new("_numax._http.example.")).is_err()); + } + + #[test] + fn transient_retry_never_outlives_the_last_valid_view() { + let now = Instant::now(); + let valid_until = now + Duration::from_secs(2); + + assert_eq!( + retry_deadline(now, Duration::from_secs(30), Some(valid_until)).unwrap(), + valid_until + ); + assert_eq!( + retry_deadline(now, Duration::from_secs(1), Some(valid_until)).unwrap(), + now + Duration::from_secs(1) + ); + } + + #[test] + fn negative_dns_ttl_is_preserved_and_bounded() { + assert_eq!( + bounded_negative_ttl(Some(30), Duration::from_secs(60)), + Duration::from_secs(30) + ); + assert_eq!( + bounded_negative_ttl(Some(120), Duration::from_secs(60)), + Duration::from_secs(60) + ); + assert_eq!( + bounded_negative_ttl(None, Duration::from_secs(60)), + Duration::from_secs(60) + ); + } + + #[test] + fn already_expired_answers_are_not_published() { + let config = DnsSrvDiscoveryConfig::new("_numax._tcp.example."); + let state = DynamicState::new(8); + state.replace(vec!["stale.example:9000".into()]); + let answer = SrvAnswer { + records: vec![SRV::new( + 0, + 0, + 9001, + Name::from_ascii("expired.example.").unwrap(), + )], + valid_until: Instant::now(), + }; + + let (valid_until, _) = apply_dns_answer(&config, &state, answer, None).unwrap(); + + assert!(valid_until.is_none()); + assert!(state.snapshot().peers().is_empty()); + } + + #[tokio::test] + async fn refresh_expires_stale_data_and_recovers_after_a_transient_error() { + let first = SRV::new(0, 0, 9001, Name::from_ascii("first.example.").unwrap()); + let second = SRV::new(0, 0, 9002, Name::from_ascii("second.example.").unwrap()); + let resolver = Arc::new(SequenceResolver { + steps: Mutex::new(VecDeque::from([ + ResolverStep::Success(vec![first], Duration::from_millis(20)), + ResolverStep::TransientFailure, + ResolverStep::Success(vec![second], Duration::from_secs(1)), + ])), + }); + let mut config = DnsSrvDiscoveryConfig::new("_numax._tcp.example."); + config.retry_interval = Duration::from_millis(10); + let provider = DnsSrvDiscovery::with_resolver(config, resolver); + let mut watch = provider.watch().await.unwrap(); + + let first = tokio::time::timeout(Duration::from_secs(1), watch.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!( + super::super::observed_peers(first.change), + ["first.example:9001"] + ); + let expired = tokio::time::timeout(Duration::from_secs(1), watch.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!( + super::super::observed_peers(expired.change), + Vec::::new() + ); + let recovered = tokio::time::timeout(Duration::from_secs(1), watch.recv()) + .await + .unwrap() + .unwrap(); + assert_eq!( + super::super::observed_peers(recovered.change), + ["second.example:9002"] + ); + + provider.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn requested_shutdown_clears_populated_snapshot_and_is_terminal() { + let resolver = Arc::new(SequenceResolver { + steps: Mutex::new(VecDeque::from([ResolverStep::Success( + vec![SRV::new( + 0, + 0, + 9000, + Name::from_ascii("cached.example.").unwrap(), + )], + Duration::from_secs(30), + )])), + }); + let provider = DnsSrvDiscovery::with_resolver( + DnsSrvDiscoveryConfig::new("_numax._tcp.example."), + resolver, + ); + let mut events = provider.watch().await.unwrap(); + assert_eq!( + super::super::next_changed_peers(&mut events, &[]).await, + ["cached.example:9000"] + ); + provider.shutdown().await.unwrap(); + provider.shutdown().await.unwrap(); + assert!(provider.inner.state.snapshot().peers().is_empty()); + assert!( + super::super::next_changed_peers(&mut events, &["cached.example:9000".into()]) + .await + .is_empty() + ); + assert!(provider.watch().await.is_err()); + assert!(provider.discover().await.is_err()); + } + + #[tokio::test] + async fn shutdown_cancels_a_stalled_lookup() { + let config = DnsSrvDiscoveryConfig::new("_numax._tcp.example."); + let provider = DnsSrvDiscovery::with_resolver(config, Arc::new(PendingResolver)); + provider.watch().await.unwrap(); + tokio::task::yield_now().await; + + tokio::time::timeout(Duration::from_secs(1), provider.shutdown()) + .await + .unwrap() + .unwrap(); + } +} diff --git a/crates/nx-core/src/discovery/dynamic.rs b/crates/nx-core/src/discovery/dynamic.rs new file mode 100644 index 0000000..0e1b7d1 --- /dev/null +++ b/crates/nx-core/src/discovery/dynamic.rs @@ -0,0 +1,818 @@ +use std::future::Future; +use std::sync::{Arc, Mutex, MutexGuard}; +use std::time::Duration; + +use tokio::sync::{broadcast, watch}; +use tokio::task::{JoinError, JoinHandle}; +use tokio::time::Instant; + +use super::{ + DiscoveryChange, DiscoveryError, DiscoveryEvent, DiscoverySnapshot, DiscoveryWatch, + MAX_DISCOVERY_EVENT_CAPACITY, +}; + +pub(super) fn checked_deadline( + now: Instant, + duration: Duration, + provider: &str, + field: &str, +) -> Result { + now.checked_add(duration) + .ok_or_else(|| DiscoveryError::Provider { + provider: provider.into(), + message: format!("{field} deadline is not representable"), + retryable: false, + }) +} + +pub(super) fn validate_durations( + provider: &str, + durations: &[(&str, Duration)], +) -> Result<(), DiscoveryError> { + let now = Instant::now(); + for (field, duration) in durations { + if now.checked_add(*duration).is_none() { + return Err(DiscoveryError::InvalidConfiguration { + provider: provider.into(), + message: format!("{field} deadline is not representable"), + }); + } + } + Ok(()) +} + +#[cfg(test)] +pub(super) fn deadline_boundary() -> Instant { + let now = Instant::now(); + // Find the platform's actual boundary, rather than inventing a cap. + let (mut low, mut high) = (0, u64::MAX); + while low < high { + let middle = low + (high - low).div_ceil(2); + if now.checked_add(Duration::from_secs(middle)).is_some() { + low = middle; + } else { + high = middle - 1; + } + } + now.checked_add(Duration::from_secs(low)).unwrap() +} + +/// One owned generation. Completion is published only after the worker was +/// joined and external cleanup finished; JoinHandle::is_finished is not a +/// restart barrier. Dropping a caller never takes ownership of this sequence. +/// The provider signals stop on Drop; the supervisor then completes its bounded +/// cleanup even if no waiter remains. Runtime teardown aborts its child task. +#[derive(Clone)] +pub(super) struct ProviderTask { + provider: &'static str, + _supervisor: Arc>, + completion: watch::Receiver>, +} + +#[derive(Clone)] +struct TaskCompletion { + result: Result<(), DiscoveryError>, + restart: Result<(), DiscoveryError>, +} + +impl ProviderTask { + pub(super) fn spawn( + provider: &'static str, + state: Arc, + shutdown: watch::Receiver, + worker: F, + cleanup: impl FnOnce() -> C + Send + 'static, + ) -> Self + where + F: Future> + Send + 'static, + C: Future> + Send + 'static, + { + let (complete, completion) = watch::channel(None); + state.activate(); + // Construct guards before spawning, including for runtime teardown. + let mut final_state = FinalizeState { state, armed: true }; + let supervisor = tokio::spawn(async move { + let mut worker = AbortOnDropJoin(tokio::spawn(worker)); + // Workers select on shutdown at blocking operations. Let them + // finish bounded transactions (notably mDNS retirement) normally. + let joined = (&mut worker.0).await; + let requested = *shutdown.borrow() || shutdown.has_changed().is_err(); + let (result, restart) = match joined { + Ok(Ok(())) => (Ok(()), Ok(())), + Ok(Err(error)) => (Err(error.clone()), Err(error)), + Err(error) => { + let error = DiscoveryError::Provider { + provider: provider.into(), + message: format!("provider task failed: {error}"), + retryable: false, + }; + // A panic is recoverable by a new generation, but is still + // reported by shutdown rather than silently discarded. + // Requested stop is cooperative, not an abort: a cancelled + // worker's JoinError must also remain observable. + (Err(error), Ok(())) + } + }; + // A stop request racing a failure does not make that failure an + // orderly exit. Existing subscribers must still be invalidated. + final_state.state.finish(!requested || result.is_err()); + // Cleanup is fallible too: catch its panic via JoinError without + // losing the completion signal or admitting an unsafe restart. + let mut cleanup = AbortOnDropJoin(tokio::spawn(async move { cleanup().await })); + let cleaned = (&mut cleanup.0).await.unwrap_or_else(|error| { + Err(DiscoveryError::Provider { + provider: provider.into(), + message: format!("provider cleanup task failed: {error}"), + retryable: false, + }) + }); + if let Err(error) = &cleaned { + tracing::warn!(%error, provider, "discovery cleanup failed"); + final_state.state.finish(true); + } + // Retryability of the worker and proof of resource retirement are + // independent. Even a retryable cleanup error cannot authorize a + // replacement that may overlap the previous external resources. + let restart = cleaned + .clone() + .map_err(|error| DiscoveryError::Provider { + provider: provider.into(), + message: format!( + "provider cleanup was not confirmed; restart blocked: {error}" + ), + retryable: false, + }) + .and(restart); + let result = result.and(cleaned); + if let Err(error) = &result { + tracing::warn!(%error, provider, "discovery generation ended"); + } + // Disarm before publishing completion: no old-generation state + // mutation is permitted after a replacement is admitted. + final_state.armed = false; + drop(final_state); + complete.send_replace(Some(TaskCompletion { result, restart })); + }); + Self { + provider, + _supervisor: Arc::new(supervisor), + completion, + } + } + + /// true means the current generation is live. During cleanup callers must + /// retry, not subscribe to a watch whose producer has already exited. + pub(super) fn running( + &self, + provider: &str, + state: &DynamicState, + ) -> Result { + // Keep the read guard through the closed-channel check. Otherwise a + // completion published between these reads could be mistaken for a + // supervisor failure merely because its sender has already dropped. + let completion = self.completion.borrow(); + if let Some(done) = completion.as_ref() { + if let Err(error) = &done.restart + && !matches!( + error, + DiscoveryError::Provider { + retryable: true, + .. + } + ) + { + return Err(error.clone()); + } + return Ok(false); + } + if self.completion.has_changed().is_err() { + return Err(DiscoveryError::Provider { + provider: provider.into(), + message: "generation supervisor failed".into(), + retryable: false, + }); + } + state.check_available()?; + Ok(true) + } + + pub(super) async fn join(mut self) -> Result<(), DiscoveryError> { + loop { + if let Some(done) = self.completion.borrow_and_update().clone() { + return done.result; + } + self.completion + .changed() + .await + .map_err(|_| DiscoveryError::Provider { + provider: self.provider.into(), + message: "generation supervisor failed".into(), + retryable: false, + })?; + } + } + + #[cfg(test)] + pub(super) fn same_generation(&self, other: &Self) -> bool { + self.completion.same_channel(&other.completion) + } + + #[cfg(test)] + pub(super) fn completion_ready(&self) -> bool { + self.completion.borrow().is_some() + } +} + +struct AbortOnDropJoin(JoinHandle); +impl Drop for AbortOnDropJoin { + fn drop(&mut self) { + self.0.abort(); + } +} + +struct FinalizeState { + state: Arc, + armed: bool, +} +impl Drop for FinalizeState { + fn drop(&mut self) { + if self.armed { + self.state.finish(true); + } + } +} + +/// Aborts a detached Tokio task if the shutdown future owning it is cancelled. +pub(crate) struct AbortOnDropTask(Option>); + +impl AbortOnDropTask { + pub(crate) fn new(task: JoinHandle<()>) -> Self { + Self(Some(task)) + } + + pub(crate) async fn join(mut self) -> Result<(), JoinError> { + let Some(task) = self.0.as_mut() else { + return Ok(()); + }; + let result = task.await; + self.0.take(); + result + } +} + +impl Drop for AbortOnDropTask { + fn drop(&mut self) { + if let Some(task) = self.0.take() { + task.abort(); + } + } +} + +/// Shared, bounded state for providers whose complete view changes over time. +pub(super) struct DynamicState { + inner: Mutex, + event_capacity: usize, + #[cfg(test)] + panic_next_observation: std::sync::atomic::AtomicBool, +} + +struct State { + available: bool, + revision: u64, + peers: Vec, + observations: Vec, + events: broadcast::Sender, +} + +impl DynamicState { + pub(super) fn new(event_capacity: usize) -> Self { + // Public provider constructors reject out-of-range capacities. Clamp + // defensively for internal callers, including later channel rotations, + // so unchecked values cannot trigger oversized allocation or overflow. + let event_capacity = event_capacity.clamp(1, MAX_DISCOVERY_EVENT_CAPACITY); + let (events, _) = broadcast::channel(event_capacity); + Self { + inner: Mutex::new(State { + available: true, + revision: 0, + peers: Vec::new(), + observations: Vec::new(), + events, + }), + event_capacity, + #[cfg(test)] + panic_next_observation: std::sync::atomic::AtomicBool::new(false), + } + } + + pub(super) fn snapshot(&self) -> DiscoverySnapshot { + let state = self.lock(); + state.snapshot() + } + + #[cfg(test)] + pub(super) fn watch(&self) -> DiscoveryWatch { + // Subscription and snapshot are captured while producers are excluded, + // so a transition cannot fall into a snapshot/watch gap. + let state = self.lock(); + let receiver = state.events.subscribe(); + DiscoveryWatch::new(state.snapshot(), receiver) + } + + pub(super) fn live_watch(&self) -> Result { + let state = self.lock(); + if !state.available { + return Err(DiscoveryError::WatchClosed); + } + Ok(DiscoveryWatch::new( + state.snapshot(), + state.events.subscribe(), + )) + } + + fn check_available(&self) -> Result<(), DiscoveryError> { + if self.lock().available { + Ok(()) + } else { + Err(DiscoveryError::WatchClosed) + } + } + + fn activate(&self) { + self.lock().available = true; + } + + fn finish(&self, unexpected: bool) { + let mut state = self.lock(); + state.available = false; + if !state.peers.is_empty() { + self.publish(&mut state, Vec::new(), Vec::new()); + } + if unexpected { + let (events, _) = broadcast::channel(self.event_capacity); + state.events = events; + } + } + + /// Replace the complete view as one revision so consumers never observe a + /// transient partial diff or lose a pure ordering change. + pub(super) fn replace(&self, peers: Vec) { + let mut state = self.lock(); + if state.peers == peers { + return; + } + let observations = peers + .iter() + .map(|peer| { + state + .peers + .iter() + .position(|old| old == peer) + .map(|index| state.observations[index]) + .unwrap_or_else(std::time::Instant::now) + }) + .collect(); + self.publish(&mut state, peers, observations); + } + + /// Successful observation of the entire view, including an identical view. + pub(super) fn observe(&self, peers: Vec) { + let now = std::time::Instant::now(); + self.observe_at(peers.into_iter().map(|peer| (peer, now)).collect()); + } + + /// Aggregate views preserve each endpoint's latest successful observation. + pub(super) fn observe_at(&self, peers: Vec<(String, std::time::Instant)>) { + #[cfg(test)] + assert!( + !self + .panic_next_observation + .swap(false, std::sync::atomic::Ordering::SeqCst), + "injected provider observation panic" + ); + let (peers, observations) = peers.into_iter().unzip(); + let mut state = self.lock(); + if state.peers == peers && state.observations == observations { + return; + } + self.publish(&mut state, peers, observations); + } + + fn publish( + &self, + state: &mut State, + peers: Vec, + observations: Vec, + ) { + let Some(revision) = state.revision.checked_add(1) else { + tracing::error!("discovery revision space exhausted; rejecting provider update"); + return; + }; + state.peers = peers; + state.observations = observations; + state.revision = revision; + let event = DiscoveryEvent { + revision: state.revision, + change: DiscoveryChange::Observed(state.snapshot()), + }; + let _ = state.events.send(event); + } + + /// Close current subscriptions while preserving the latest snapshot for a + /// fresh watch after a provider-level restart. + #[cfg(test)] + pub(super) fn invalidate_watches(&self) { + let mut state = self.lock(); + let (events, _) = broadcast::channel(self.event_capacity); + state.events = events; + } + + fn lock(&self) -> MutexGuard<'_, State> { + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + } + + #[cfg(test)] + pub(super) fn panic_on_next_observation(&self) { + self.panic_next_observation + .store(true, std::sync::atomic::Ordering::SeqCst); + } +} + +impl State { + fn snapshot(&self) -> DiscoverySnapshot { + DiscoverySnapshot::observed( + self.revision, + self.peers + .iter() + .cloned() + .zip(self.observations.iter().copied()) + .collect(), + ) + } +} + +#[cfg(test)] +pub(super) async fn assert_invalidated(watch: &mut DiscoveryWatch) { + tokio::time::timeout(Duration::from_secs(2), async { + let mut cleared = false; + loop { + match watch.recv().await { + Ok(event) => { + if super::observed_peers(event.change).is_empty() { + cleared = true; + } + } + Err(DiscoveryError::WatchClosed) => { + assert!(cleared); + break; + } + other => panic!("unexpected watch result: {other:?}"), + } + } + }) + .await + .unwrap(); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::discovery::DiscoveryError; + use std::sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }; + use std::time::Duration; + + #[tokio::test] + async fn internal_event_capacity_clamp_survives_channel_rotation() { + for (capacity, normalized) in [ + (0, 1), + (1, 1), + (MAX_DISCOVERY_EVENT_CAPACITY, MAX_DISCOVERY_EVENT_CAPACITY), + ( + MAX_DISCOVERY_EVENT_CAPACITY + 1, + MAX_DISCOVERY_EVENT_CAPACITY, + ), + (usize::MAX, MAX_DISCOVERY_EVENT_CAPACITY), + ] { + let state = DynamicState::new(capacity); + assert_eq!(state.event_capacity, normalized); + for _ in 0..2 { + let mut events = state.watch(); + for revision in 0..=normalized { + state.replace(vec![format!("peer-{revision}:9000")]); + } + assert_eq!( + events.recv().await, + Err(DiscoveryError::WatchOverflow { missed: 1 }) + ); + state.invalidate_watches(); + } + } + } + + #[test] + fn representable_duration_can_overflow_only_after_the_clock_advances() { + let last = deadline_boundary(); + let one_second = Duration::from_secs(1); + assert!(validate_durations("test", &[("delay", one_second)]).is_ok()); + assert!(checked_deadline(last - one_second, one_second, "test", "delay").is_ok()); + assert!(matches!( + checked_deadline(last, one_second, "test", "delay"), + Err(DiscoveryError::Provider { + retryable: false, + .. + }) + )); + } + + #[tokio::test] + async fn dropping_a_shutdown_waiter_does_not_cancel_owned_cleanup() { + let state = Arc::new(DynamicState::new(8)); + state.observe(vec!["cached:9000".into()]); + let (stop, stop_rx) = watch::channel(true); + let (release, released) = tokio::sync::oneshot::channel::<()>(); + let owner = ProviderTask::spawn( + "test", + state.clone(), + stop_rx, + async { Ok(()) }, + move || async move { + released.await.unwrap(); + Ok(()) + }, + ); + let mut waiter = Box::pin(owner.clone().join()); + std::future::poll_fn(|cx| { + assert!(waiter.as_mut().poll(cx).is_pending()); + std::task::Poll::Ready(()) + }) + .await; + drop(waiter); + release.send(()).unwrap(); + owner.join().await.unwrap(); + assert!(state.snapshot().peers().is_empty()); + drop(stop); + } + + #[tokio::test] + async fn requested_stop_does_not_hide_worker_errors_or_panics_from_watches() { + for panic in [false, true] { + let state = Arc::new(DynamicState::new(8)); + state.observe(vec!["cached:9000".into()]); + let mut events = state.watch(); + let (_stop, stop_rx) = watch::channel(true); + let task = ProviderTask::spawn( + "test", + state.clone(), + stop_rx, + async move { + assert!(!panic, "injected panic during requested shutdown"); + Err(DiscoveryError::Provider { + provider: "test".into(), + message: "worker failed during requested shutdown".into(), + retryable: true, + }) + }, + || async { Ok(()) }, + ); + let error = task.join().await.unwrap_err(); + if panic { + assert!(matches!(error, DiscoveryError::Provider { message, .. } + if message.contains("provider task failed") && message.contains("panic"))); + } + assert!(state.snapshot().peers().is_empty()); + assert_invalidated(&mut events).await; + } + } + + #[tokio::test] + async fn failed_cleanup_is_a_terminal_restart_barrier_even_for_retryable_errors() { + for requested in [false, true] { + let state = Arc::new(DynamicState::new(8)); + state.observe(vec!["cached:9000".into()]); + let mut events = state.watch(); + let (_stop, stop_rx) = watch::channel(requested); + let error = DiscoveryError::Provider { + provider: "test".into(), + message: "external cleanup not confirmed".into(), + retryable: true, + }; + let cleanup_error = error.clone(); + let task = ProviderTask::spawn( + "test", + state.clone(), + stop_rx, + async { Ok(()) }, + move || async move { Err(cleanup_error) }, + ); + assert_eq!(task.clone().join().await, Err(error)); + assert!(matches!( + task.running("test", &state), + Err(DiscoveryError::Provider { + retryable: false, + .. + }) + )); + assert!(state.live_watch().is_err()); + assert_invalidated(&mut events).await; + } + } + + #[tokio::test] + async fn successful_requested_stop_clears_without_invalidating_existing_watch() { + let state = Arc::new(DynamicState::new(8)); + state.observe(vec!["cached:9000".into()]); + let mut events = state.watch(); + let (_stop, stop_rx) = watch::channel(true); + let task = + ProviderTask::spawn("test", state.clone(), stop_rx, async { Ok(()) }, || async { + Ok(()) + }); + task.join().await.unwrap(); + assert!(super::super::observed_peers(events.recv().await.unwrap().change).is_empty()); + assert!(state.snapshot().peers().is_empty()); + assert!(state.live_watch().is_err()); + let mut next = Box::pin(events.recv()); + std::future::poll_fn(|cx| { + assert!(next.as_mut().poll(cx).is_pending()); + std::task::Poll::Ready(()) + }) + .await; + } + + #[tokio::test] + async fn cleanup_panic_completes_with_join_error_and_prevents_restart() { + let state = Arc::new(DynamicState::new(8)); + state.observe(vec!["cached:9000".into()]); + let mut events = state.watch(); + let (_stop, stop_rx) = watch::channel(false); + let owner = + ProviderTask::spawn("test", state.clone(), stop_rx, async { Ok(()) }, || async { + panic!("injected cleanup panic"); + }); + assert_invalidated(&mut events).await; + let result = tokio::time::timeout(Duration::from_secs(1), owner.clone().join()) + .await + .unwrap(); + assert!( + matches!(result, Err(DiscoveryError::Provider { message, retryable: false, .. }) + if message.contains("cleanup task failed") && message.contains("panic")) + ); + assert!(owner.running("test", &state).is_err()); + assert!(state.live_watch().is_err()); + assert!(state.snapshot().peers().is_empty()); + } + + #[tokio::test] + async fn completion_not_joinhandle_finished_is_the_restart_barrier() { + let state = Arc::new(DynamicState::new(8)); + state.observe(vec!["cached:9000".into()]); + let mut events = state.watch(); + let (_stop, stop_rx) = watch::channel(false); + let (release, released) = tokio::sync::oneshot::channel(); + let owner = ProviderTask::spawn( + "test", + state.clone(), + stop_rx, + async { + panic!("worker panic"); + }, + move || async move { + released.await.unwrap(); + Ok(()) + }, + ); + assert_invalidated(&mut events).await; + assert!(owner.running("test", &state).is_err()); + assert!(state.live_watch().is_err()); + release.send(()).unwrap(); + assert!(owner.clone().join().await.is_err()); + // Pretend the supervisor has not returned from its final poll yet: + // completion is sufficient because it cannot mutate state afterwards. + assert!(!owner.running("test", &state).unwrap()); + state.activate(); + state.observe(vec!["replacement:9000".into()]); + tokio::task::yield_now().await; + assert_eq!(state.snapshot().peers(), ["replacement:9000"]); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn completion_publication_and_sender_drop_do_not_report_supervisor_failure() { + tokio::time::timeout(Duration::from_secs(3), async { + for _ in 0..128 { + let state = Arc::new(DynamicState::new(8)); + let (_stop, stop_rx) = watch::channel(false); + let owner = ProviderTask::spawn( + "test", + state.clone(), + stop_rx, + async { Ok(()) }, + || async { Ok(()) }, + ); + loop { + match owner.running("test", &state) { + Ok(false) => break, + Ok(true) | Err(DiscoveryError::WatchClosed) => { + tokio::task::yield_now().await + } + other => panic!("completion race reported a failure: {other:?}"), + } + } + owner.join().await.unwrap(); + } + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn identical_fresh_observations_advance_but_cached_republication_does_not() { + let state = DynamicState::new(2); + let now = std::time::Instant::now(); + state.observe_at(vec![("a:1".into(), now)]); + let mut watch = state.watch(); + let snapshot = watch.snapshot().clone(); + state.replace(vec!["a:1".into()]); + assert_eq!(state.snapshot(), snapshot); + state.observe_at(vec![("a:1".into(), now + Duration::from_secs(1))]); + let event = watch.recv().await.unwrap(); + assert_eq!(event.revision, snapshot.revision() + 1); + let DiscoveryChange::Observed(ref fresh) = event.change else { + panic!("missing observation"); + }; + assert_eq!(fresh.peers(), snapshot.peers()); + assert_ne!(fresh.observations(), snapshot.observations()); + assert_eq!(fresh, state.watch().snapshot()); + for offset in 2..8 { + state.observe_at(vec![("a:1".into(), now + Duration::from_secs(offset))]); + } + assert!(matches!( + watch.recv().await, + Err(DiscoveryError::WatchOverflow { .. }) + )); + assert_eq!(state.snapshot(), *state.watch().snapshot()); + } + + #[tokio::test] + async fn replacement_has_a_contiguous_watch_stream() { + let state = DynamicState::new(8); + state.replace(vec!["a:1".into()]); + let mut watch = state.watch(); + state.replace(vec!["b:2".into(), "c:3".into()]); + + let event = watch.recv().await.unwrap(); + assert_eq!(event.revision, 2); + assert_eq!(super::super::observed_peers(event.change), ["b:2", "c:3"]); + assert_eq!(state.snapshot().peers(), ["b:2", "c:3"]); + } + + #[tokio::test] + async fn invalidation_closes_existing_watches_and_preserves_the_snapshot() { + let state = DynamicState::new(8); + state.replace(vec!["a:1".into()]); + let mut old_watch = state.watch(); + + state.invalidate_watches(); + + assert_eq!( + old_watch.recv().await.unwrap_err(), + DiscoveryError::WatchClosed + ); + let fresh_watch = state.watch(); + assert_eq!(fresh_watch.snapshot().revision(), 1); + assert_eq!(fresh_watch.snapshot().peers(), ["a:1"]); + } + + #[tokio::test] + async fn dropping_an_owned_shutdown_handle_aborts_the_task() { + struct Dropped(Arc); + impl Drop for Dropped { + fn drop(&mut self) { + self.0.store(true, Ordering::SeqCst); + } + } + + let dropped = Arc::new(AtomicBool::new(false)); + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let task_dropped = Arc::clone(&dropped); + let task = tokio::spawn(async move { + let _guard = Dropped(task_dropped); + let _ = started_tx.send(()); + std::future::pending::<()>().await; + }); + started_rx.await.unwrap(); + + drop(AbortOnDropTask::new(task)); + tokio::time::timeout(Duration::from_secs(1), async { + while !dropped.load(Ordering::SeqCst) { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + } +} diff --git a/crates/nx-core/src/discovery/file_watch.rs b/crates/nx-core/src/discovery/file_watch.rs new file mode 100644 index 0000000..ff66a87 --- /dev/null +++ b/crates/nx-core/src/discovery/file_watch.rs @@ -0,0 +1,541 @@ +use std::collections::HashSet; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex as StdMutex}; +use std::time::Duration; + +use async_trait::async_trait; +use tokio::io::AsyncReadExt; +use tokio::sync::watch; +use tokio::time::Instant; + +use super::dynamic::{DynamicState, ProviderTask, checked_deadline, validate_durations}; +use super::{ + DEFAULT_DISCOVERY_CLUSTER, DEFAULT_DISCOVERY_EVENT_CAPACITY, DEFAULT_MAX_PEER_CANDIDATES, + DiscoveryError, DiscoverySnapshot, DiscoveryWatch, PeerAnnouncement, PeerDiscovery, + validate_event_capacity, +}; + +const PROVIDER: &str = "file"; +const DEFAULT_POLL_INTERVAL: Duration = Duration::from_secs(2); +const DEFAULT_MAX_FILE_BYTES: usize = 1024 * 1024; + +/// Limits and polling policy for [`FileWatchDiscovery`]. +#[derive(Debug, Clone)] +pub struct FileWatchDiscoveryConfig { + pub path: PathBuf, + pub cluster_id: String, + pub poll_interval: Duration, + pub max_file_bytes: usize, + pub max_candidates: usize, + /// Event channel capacity in `1..=super::MAX_DISCOVERY_EVENT_CAPACITY`. + /// Defaults to [`DEFAULT_DISCOVERY_EVENT_CAPACITY`]; validated by the provider constructor. + pub event_capacity: usize, +} + +impl FileWatchDiscoveryConfig { + pub fn new(path: impl Into) -> Self { + Self { + path: path.into(), + cluster_id: DEFAULT_DISCOVERY_CLUSTER.to_string(), + poll_interval: DEFAULT_POLL_INTERVAL, + max_file_bytes: DEFAULT_MAX_FILE_BYTES, + max_candidates: DEFAULT_MAX_PEER_CANDIDATES, + event_capacity: DEFAULT_DISCOVERY_EVENT_CAPACITY, + } + } +} + +struct Lifecycle { + stopped: bool, + shutdown: Option>, + task: Option, +} + +struct Inner { + config: FileWatchDiscoveryConfig, + state: Arc, + lifecycle: StdMutex, +} + +impl Drop for Inner { + fn drop(&mut self) { + let lifecycle = self + .lifecycle + .get_mut() + .unwrap_or_else(|error| error.into_inner()); + if let Some(shutdown) = lifecycle.shutdown.take() { + let _ = shutdown.send(true); + } + } +} + +/// Watches an externally managed UTF-8 peer file. +/// +/// Each non-empty line is one endpoint; leading/trailing whitespace is removed +/// and lines beginning with `#` are comments. Updates are accepted atomically: +/// an unreadable, oversized, non-UTF-8, or over-limit version leaves the last +/// valid snapshot in place. A missing file is a valid empty snapshot, which +/// supports Kubernetes-style atomic replacement and delayed creation. +pub struct FileWatchDiscovery { + inner: Arc, +} + +impl FileWatchDiscovery { + #[cfg(test)] + pub(crate) fn panic_on_next_observation(&self) { + self.inner.state.panic_on_next_observation(); + } + + pub fn new(config: FileWatchDiscoveryConfig) -> Result { + validate_config(&config)?; + Ok(Self { + inner: Arc::new(Inner { + state: Arc::new(DynamicState::new(config.event_capacity)), + config, + lifecycle: StdMutex::new(Lifecycle { + stopped: false, + shutdown: None, + task: None, + }), + }), + }) + } + + async fn ensure_started(&self) -> Result<(), DiscoveryError> { + { + let lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + if lifecycle.stopped { + return Err(provider_error("provider is shut down", false)); + } + if let Some(task) = lifecycle.task.as_ref() + && task.running(PROVIDER, &self.inner.state)? + { + return Ok(()); + } + } + + let initial = read_peer_file(&self.inner.config).await?; + let mut lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + if lifecycle.stopped { + return Err(provider_error("provider is shut down", false)); + } + if let Some(task) = lifecycle.task.as_ref() + && task.running(PROVIDER, &self.inner.state)? + { + return Ok(()); + } + self.inner.state.observe(initial); + let (shutdown, shutdown_rx) = watch::channel(false); + let config = self.inner.config.clone(); + let state = Arc::clone(&self.inner.state); + lifecycle.shutdown = Some(shutdown); + lifecycle.task = Some(ProviderTask::spawn( + PROVIDER, + state.clone(), + shutdown_rx.clone(), + run_file_watch(config, state, shutdown_rx), + || async { Ok(()) }, + )); + Ok(()) + } +} + +#[async_trait] +impl PeerDiscovery for FileWatchDiscovery { + fn cluster_id(&self) -> &str { + &self.inner.config.cluster_id + } + + async fn discover(&self) -> Result { + self.ensure_started().await?; + Ok(self.inner.state.snapshot()) + } + + async fn announce(&self, _announcement: &PeerAnnouncement) -> Result<(), DiscoveryError> { + Err(DiscoveryError::Unsupported { + provider: PROVIDER.to_string(), + operation: "announcement", + }) + } + + async fn watch(&self) -> Result { + self.ensure_started().await?; + self.inner.state.live_watch() + } + + fn request_shutdown(&self) { + let mut lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + lifecycle.stopped = true; + if let Some(shutdown) = lifecycle.shutdown.as_ref() { + let _ = shutdown.send(true); + } + } + + async fn shutdown(&self) -> Result<(), DiscoveryError> { + self.request_shutdown(); + let task = { + let lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + lifecycle.task.clone() + }; + match task { + Some(task) => task.join().await, + None => Ok(()), + } + } +} + +async fn run_file_watch( + config: FileWatchDiscoveryConfig, + state: Arc, + mut shutdown: watch::Receiver, +) -> Result<(), DiscoveryError> { + // The initial view was loaded by ensure_started(). + let mut next = checked_deadline( + Instant::now(), + config.poll_interval, + PROVIDER, + "poll_interval", + )?; + loop { + if *shutdown.borrow() || shutdown.has_changed().is_err() { + return Ok(()); + } + tokio::select! { + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return Ok(()); + } + } + _ = tokio::time::sleep_until(next) => { + tokio::select! { + changed = shutdown.changed() => { + if changed.is_err() || *shutdown.borrow() { + return Ok(()); + } + } + result = read_peer_file(&config) => match result { + Ok(peers) => state.observe(peers), + Err(error) => tracing::warn!(%error, path = %config.path.display(), "ignoring invalid peer file update"), + } + } + next = next_poll_deadline(next, Instant::now(), config.poll_interval)?; + } + } + } +} + +fn next_poll_deadline( + previous: Instant, + now: Instant, + period: Duration, +) -> Result { + let next = checked_deadline(previous, period, PROVIDER, "poll_interval")?; + if next > now { + Ok(next) + } else { + // Skip missed polls without relying on Interval's unchecked addition. + checked_deadline(now, period, PROVIDER, "poll_interval") + } +} + +async fn read_peer_file(config: &FileWatchDiscoveryConfig) -> Result, DiscoveryError> { + let file = match tokio::fs::File::open(&config.path).await { + Ok(file) => file, + Err(error) if error.kind() == ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(io_error(&config.path, error)), + }; + let limit = u64::try_from(config.max_file_bytes) + .unwrap_or(u64::MAX) + .saturating_add(1); + let mut bytes = Vec::new(); + file.take(limit) + .read_to_end(&mut bytes) + .await + .map_err(|error| io_error(&config.path, error))?; + if bytes.len() > config.max_file_bytes { + return Err(provider_error( + format!( + "{} exceeds the {} byte limit", + config.path.display(), + config.max_file_bytes + ), + true, + )); + } + let contents = String::from_utf8(bytes).map_err(|_| { + provider_error( + format!("{} is not valid UTF-8", config.path.display()), + true, + ) + })?; + parse_peer_file(&contents, config.max_candidates) +} + +fn parse_peer_file(contents: &str, max_candidates: usize) -> Result, DiscoveryError> { + let mut seen = HashSet::new(); + let mut peers = Vec::new(); + for (index, line) in contents.lines().enumerate() { + let endpoint = line.trim(); + if endpoint.is_empty() || endpoint.starts_with('#') { + continue; + } + let endpoint = crate::sync_manager::canonicalize_endpoint(endpoint).map_err(|error| { + provider_error(format!("line {} is invalid: {error}", index + 1), true) + })?; + if seen.insert(endpoint.clone()) { + if peers.len() == max_candidates { + return Err(provider_error( + format!("peer file exceeds the {max_candidates} candidate limit"), + true, + )); + } + peers.push(endpoint); + } + } + Ok(peers) +} + +fn validate_config(config: &FileWatchDiscoveryConfig) -> Result<(), DiscoveryError> { + validate_event_capacity(PROVIDER, config.event_capacity)?; + if config.path.as_os_str().is_empty() { + return Err(invalid("path must not be empty")); + } + if config.cluster_id.trim().is_empty() { + return Err(invalid("cluster_id must not be empty")); + } + if config.poll_interval.is_zero() { + return Err(invalid("poll_interval must be greater than zero")); + } + if config.max_file_bytes == 0 || config.max_candidates == 0 { + return Err(invalid("limits must be greater than zero")); + } + validate_durations(PROVIDER, &[("poll_interval", config.poll_interval)]) +} + +fn invalid(message: impl Into) -> DiscoveryError { + DiscoveryError::InvalidConfiguration { + provider: PROVIDER.to_string(), + message: message.into(), + } +} + +fn provider_error(message: impl Into, retryable: bool) -> DiscoveryError { + DiscoveryError::Provider { + provider: PROVIDER.to_string(), + message: message.into(), + retryable, + } +} + +fn io_error(path: &Path, error: std::io::Error) -> DiscoveryError { + provider_error(format!("cannot read {}: {error}", path.display()), true) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn extreme_poll_interval_is_rejected_before_file_io_or_spawning() { + let mut config = FileWatchDiscoveryConfig::new("unused-peers"); + config.poll_interval = Duration::MAX; + assert!(matches!(FileWatchDiscovery::new(config), + Err(DiscoveryError::InvalidConfiguration { provider, message }) + if provider == PROVIDER && message.contains("poll_interval"))); + } + + #[test] + fn runtime_poll_overflow_is_not_an_immediate_retry() { + let now = Instant::now(); + assert!(matches!( + next_poll_deadline(now, now, Duration::MAX), + Err(DiscoveryError::Provider { + retryable: false, + .. + }) + )); + assert_eq!( + next_poll_deadline(now, now + Duration::from_secs(10), Duration::from_secs(1)).unwrap(), + now + Duration::from_secs(11) + ); + let boundary = super::super::dynamic::deadline_boundary(); + let period = Duration::from_secs(1); + assert!(next_poll_deadline(boundary, boundary, period).is_err()); + assert!(next_poll_deadline(boundary - period, boundary, period).is_err()); + } + + #[tokio::test] + async fn panicked_watch_is_reported_after_clearing_snapshot() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("peers"); + tokio::fs::write(&path, "cached.example:9000") + .await + .unwrap(); + let mut config = FileWatchDiscoveryConfig::new(path); + config.poll_interval = Duration::from_millis(10); + let provider = FileWatchDiscovery::new(config).unwrap(); + let mut events = provider.watch().await.unwrap(); + assert_eq!(events.snapshot().peers(), ["cached.example:9000"]); + provider.inner.state.panic_on_next_observation(); + super::super::dynamic::assert_invalidated(&mut events).await; + let result = provider.shutdown().await; + assert!( + matches!(result, Err(DiscoveryError::Provider { retryable: false, message, .. }) + if message.contains("provider task failed") && message.contains("panic")) + ); + assert!(provider.inner.state.snapshot().peers().is_empty()); + assert!(provider.inner.state.watch().snapshot().peers().is_empty()); + assert!(provider.watch().await.is_err()); + } + + #[tokio::test] + async fn panic_invalidates_and_concurrent_subscribers_restart_one_file_generation() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("peers"); + tokio::fs::write(&path, "cached.example:9000") + .await + .unwrap(); + let mut config = FileWatchDiscoveryConfig::new(path); + config.poll_interval = Duration::from_millis(10); + let provider = FileWatchDiscovery::new(config).unwrap(); + let mut events = provider.watch().await.unwrap(); + let old = provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .clone() + .unwrap(); + provider.inner.state.panic_on_next_observation(); + super::super::dynamic::assert_invalidated(&mut events).await; + assert!(old.clone().join().await.is_err()); + assert!(provider.inner.state.snapshot().peers().is_empty()); + let (first, second) = tokio::join!(provider.watch(), provider.watch()); + assert_eq!(first.unwrap().snapshot().peers(), ["cached.example:9000"]); + assert_eq!(second.unwrap().snapshot().peers(), ["cached.example:9000"]); + let current = provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .clone() + .unwrap(); + assert!(!old.same_generation(¤t)); + provider.watch().await.unwrap(); + assert!( + current.same_generation( + provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .as_ref() + .unwrap() + ) + ); + provider.shutdown().await.unwrap(); + provider.shutdown().await.unwrap(); + assert!(provider.inner.state.snapshot().peers().is_empty()); + assert!(provider.watch().await.is_err()); + assert!(provider.discover().await.is_err()); + } + + async fn replace_file_bytes(path: &Path, bytes: &[u8]) { + let staging = path.with_extension("staging"); + tokio::fs::write(&staging, bytes).await.unwrap(); + tokio::fs::rename(staging, path).await.unwrap(); + } + + async fn replace_file(path: &Path, contents: &str) { + replace_file_bytes(path, contents.as_bytes()).await; + } + + #[test] + fn parser_preserves_order_and_deduplicates() { + let peers = parse_peer_file("# peers\n b:2 \na:1\nb:2\n", 2).unwrap(); + assert_eq!(peers, ["b:2", "a:1"]); + } + + #[test] + fn parser_rejects_the_whole_over_limit_update() { + assert!(parse_peer_file("a:1\nb:2\n", 1).is_err()); + } + + #[tokio::test] + async fn missing_file_is_an_empty_initial_snapshot_and_shutdown_is_idempotent() { + let directory = tempfile::tempdir().unwrap(); + let discovery = FileWatchDiscovery::new(FileWatchDiscoveryConfig::new( + directory.path().join("peers"), + )) + .unwrap(); + assert!(discovery.discover().await.unwrap().peers().is_empty()); + discovery.shutdown().await.unwrap(); + discovery.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn watch_applies_complete_files_retains_last_good_and_stops_on_shutdown() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("peers"); + let mut config = FileWatchDiscoveryConfig::new(&path); + config.poll_interval = Duration::from_millis(10); + let discovery = FileWatchDiscovery::new(config).unwrap(); + let mut watch = discovery.watch().await.unwrap(); + assert!(watch.snapshot().peers().is_empty()); + + replace_file(&path, "b.example:2\na.example:1\n").await; + let peers = super::super::next_changed_peers(&mut watch, &[]).await; + assert_eq!(peers, ["b.example:2", "a.example:1"]); + + replace_file(&path, "valid.example:3\nnot-an-endpoint\n").await; + assert!(read_peer_file(&discovery.inner.config).await.is_err()); + + replace_file_bytes(&path, &[0xff, 0xfe]).await; + assert!(read_peer_file(&discovery.inner.config).await.is_err()); + replace_file(&path, "recovered.example:5\n").await; + let recovered = super::super::next_changed_peers(&mut watch, &peers).await; + assert_eq!(recovered, ["recovered.example:5"]); + + tokio::fs::remove_file(&path).await.unwrap(); + assert!( + super::super::next_changed_peers(&mut watch, &recovered) + .await + .is_empty() + ); + + discovery.shutdown().await.unwrap(); + let stopped_revision = discovery.inner.state.snapshot().revision(); + replace_file(&path, "late.example:4\n").await; + assert!( + tokio::time::timeout(Duration::from_millis(40), async { + loop { + // Queued observations from before shutdown remain valid; + // no event may have been produced after the final revision. + assert!(watch.recv().await.unwrap().revision <= stopped_revision); + } + }) + .await + .is_err() + ); + } +} diff --git a/crates/nx-core/src/discovery/mdns.rs b/crates/nx-core/src/discovery/mdns.rs new file mode 100644 index 0000000..92984bd --- /dev/null +++ b/crates/nx-core/src/discovery/mdns.rs @@ -0,0 +1,2342 @@ +use std::collections::{BTreeSet, HashMap}; +use std::net::{IpAddr, SocketAddr}; +use std::sync::{Arc, Mutex as StdMutex}; +use std::time::{Duration, Instant as StdInstant}; + +use async_trait::async_trait; +use mdns_sd::{ + DaemonEvent, DaemonStatus, DnsNameChange, RRType, ServiceDaemon, ServiceEvent, ServiceInfo, +}; +use tokio::sync::{mpsc, oneshot, watch}; + +use super::dynamic::{DynamicState, ProviderTask}; +use super::{ + AnnouncementSupport, DEFAULT_DISCOVERY_CLUSTER, DEFAULT_DISCOVERY_EVENT_CAPACITY, + DEFAULT_MAX_PEER_CANDIDATES, DiscoveryError, DiscoverySnapshot, DiscoveryWatch, + PeerAnnouncement, PeerDiscovery, validate_event_capacity, +}; + +const PROVIDER: &str = "mdns"; +const SERVICE_BASE: &str = "_numax._tcp.local."; +const DEFAULT_MAX_INSTANCES: usize = 1024; +const SHUTDOWN_BUDGET: Duration = Duration::from_secs(4); +const MAX_OWN_HISTORY: usize = 1024; + +struct AnnounceRequest { + endpoint: String, + reply: oneshot::Sender>, +} + +/// LAN mDNS discovery and announcement limits. +#[derive(Debug, Clone)] +pub struct MdnsDiscoveryConfig { + pub instance_name: String, + pub cluster_id: String, + pub max_instances: usize, + pub max_candidates: usize, + /// Event and announcement channel capacity in `1..=super::MAX_DISCOVERY_EVENT_CAPACITY`. + /// Defaults to [`DEFAULT_DISCOVERY_EVENT_CAPACITY`]; validated by the provider constructor. + pub event_capacity: usize, +} + +impl MdnsDiscoveryConfig { + pub fn new(instance_name: impl Into) -> Self { + Self { + instance_name: instance_name.into(), + cluster_id: DEFAULT_DISCOVERY_CLUSTER.to_string(), + max_instances: DEFAULT_MAX_INSTANCES, + max_candidates: DEFAULT_MAX_PEER_CANDIDATES, + event_capacity: DEFAULT_DISCOVERY_EVENT_CAPACITY, + } + } +} + +struct Lifecycle { + stopped: bool, + shutdown: Option, + task: Option, + announcements: Option>, +} + +struct ShutdownRequest { + requested: watch::Sender, + deadline: watch::Sender>, +} + +struct MdnsGeneration { + shutdown: ShutdownRequest, + announcements: mpsc::Sender, + task: ProviderTask, +} + +struct Inner { + config: MdnsDiscoveryConfig, + service_type: String, + state: Arc, + own_endpoint: Arc>>, + lifecycle: StdMutex, +} + +impl Drop for Inner { + fn drop(&mut self) { + let lifecycle = self + .lifecycle + .get_mut() + .unwrap_or_else(|error| error.into_inner()); + if let Some(shutdown) = lifecycle.shutdown.take() { + request_shutdown(&shutdown, SHUTDOWN_BUDGET); + } + // The browse task owns the bounded withdrawal sequence. Do not abort + // it when its caller is dropped: it must still consume both ACKs. + } +} + +/// Discovers and advertises Numax endpoints on the local multicast domain. +/// +/// mDNS instance names, TXT data, and addresses are routing hints only. They +/// never become peer identity or authorization evidence. +pub struct MdnsDiscovery { + inner: Arc, +} + +impl MdnsDiscovery { + pub fn new(config: MdnsDiscoveryConfig) -> Result { + validate_config(&config)?; + Ok(Self { + inner: Arc::new(Inner { + service_type: cluster_service_type(&config.cluster_id), + state: Arc::new(DynamicState::new(config.event_capacity)), + own_endpoint: Arc::new(StdMutex::new(None)), + config, + lifecycle: StdMutex::new(Lifecycle { + stopped: false, + shutdown: None, + task: None, + announcements: None, + }), + }), + }) + } + + fn ensure_started(&self) -> Result<(), DiscoveryError> { + self.ensure_started_with(|| self.start_generation()) + } + + fn ensure_started_with( + &self, + start: impl FnOnce() -> Result, + ) -> Result<(), DiscoveryError> { + let mut lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + if lifecycle.stopped { + return Err(provider_error("provider is shut down", false)); + } + if let Some(task) = lifecycle.task.as_ref() + && task.running(PROVIDER, &self.inner.state)? + { + return Ok(()); + } + + let generation = start()?; + lifecycle.shutdown = Some(generation.shutdown); + lifecycle.announcements = Some(generation.announcements); + lifecycle.task = Some(generation.task); + Ok(()) + } + + fn start_generation(&self) -> Result { + let daemon = ServiceDaemon::new() + .map_err(|error| provider_error(format!("cannot start mDNS daemon: {error}"), false))?; + let monitor = match daemon.monitor() { + Ok(monitor) => monitor, + Err(error) => { + let _ = daemon.shutdown(); + return Err(provider_error( + format!("cannot monitor mDNS daemon: {error}"), + true, + )); + } + }; + let events = match daemon.browse(&self.inner.service_type) { + Ok(events) => events, + Err(error) => { + let _ = daemon.shutdown(); + return Err(provider_error( + format!("cannot browse mDNS service: {error}"), + true, + )); + } + }; + let mut owned = OwnedAnnouncements::default(); + if let Some(endpoint) = self + .inner + .own_endpoint + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone() + { + let (service, fullname) = + build_service(&self.inner.config, &self.inner.service_type, &endpoint)?; + if let Err(error) = daemon.register(service) { + let _ = daemon.stop_browse(&self.inner.service_type); + let _ = daemon.shutdown(); + return Err(provider_error( + format!("cannot restore mDNS announcement: {error}"), + true, + )); + } + owned.accept(fullname, endpoint); + } + let (shutdown, shutdown_rx, shutdown_deadline_rx) = shutdown_channels(); + let (announcements_tx, announcements_rx) = mpsc::channel(self.inner.config.event_capacity); + let config = self.inner.config.clone(); + let state = Arc::clone(&self.inner.state); + let own_endpoint = Arc::clone(&self.inner.own_endpoint); + // Construct the guard before spawning: cancellation before the first + // task poll must still release the external daemon. + let cleanup = DaemonCleanup { + daemon: LiveDaemon { + daemon, + service_type: self.inner.service_type.clone(), + }, + owned, + finished: false, + }; + let task = start_mdns_task( + config, + state, + own_endpoint, + events, + monitor, + cleanup, + announcements_rx, + shutdown_rx, + shutdown_deadline_rx, + ); + Ok(MdnsGeneration { + shutdown, + announcements: announcements_tx, + task, + }) + } + + fn request_shutdown_with_budget(&self, budget: Duration) { + let mut lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + lifecycle.stopped = true; + if let Some(shutdown) = &lifecycle.shutdown { + request_shutdown(shutdown, budget); + } + self.inner + .own_endpoint + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); + } +} + +#[async_trait] +impl PeerDiscovery for MdnsDiscovery { + fn cluster_id(&self) -> &str { + &self.inner.config.cluster_id + } + + fn announcement_support(&self) -> AnnouncementSupport { + AnnouncementSupport::Required + } + + async fn discover(&self) -> Result { + self.ensure_started()?; + Ok(self.inner.state.snapshot()) + } + + async fn announce(&self, announcement: &PeerAnnouncement) -> Result<(), DiscoveryError> { + self.ensure_started()?; + let endpoint = crate::sync_manager::canonicalize_endpoint(&announcement.endpoint) + .map_err(|error| provider_error(error.to_string(), false))?; + let (reply, response) = oneshot::channel(); + { + let lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + if lifecycle.stopped { + return Err(provider_error("provider is shut down", false)); + } + lifecycle + .announcements + .as_ref() + .ok_or_else(|| provider_error("mDNS daemon is unavailable", true))? + .try_send(AnnounceRequest { endpoint, reply }) + .map_err(|error| { + provider_error(format!("cannot queue mDNS announcement: {error}"), true) + })?; + } + // Once queued, the browse task owns the transaction, even if this + // waiter is cancelled. It also serializes NameChange and shutdown. + response + .await + .map_err(|_| provider_error("mDNS announcement task stopped", true))? + } + + async fn watch(&self) -> Result { + self.ensure_started()?; + self.inner.state.live_watch() + } + + fn request_shutdown(&self) { + self.request_shutdown_with_budget(SHUTDOWN_BUDGET); + } + + async fn shutdown(&self) -> Result<(), DiscoveryError> { + self.request_shutdown(); + let completion = { + let lifecycle = self + .inner + .lifecycle + .lock() + .unwrap_or_else(|error| error.into_inner()); + lifecycle.task.clone() + }; + match completion { + Some(task) => task.join().await, + None => Ok(()), + } + } +} + +fn request_shutdown(shutdown: &ShutdownRequest, budget: Duration) { + // Publish the stop intent before the worker-visible deadline. Otherwise the + // worker could exit between the two notifications and be misclassified by + // the supervisor as an unexpected termination. + shutdown.requested.send_replace(true); + if shutdown.deadline.borrow().is_none() { + shutdown.deadline.send_replace(Some(deadline_after(budget))); + } +} + +fn deadline_after(budget: Duration) -> tokio::time::Instant { + let now = tokio::time::Instant::now(); + now.checked_add(budget).unwrap_or(now) +} + +fn shutdown_channels() -> ( + ShutdownRequest, + watch::Receiver, + watch::Receiver>, +) { + let (requested, requested_rx) = watch::channel(false); + let (deadline, deadline_rx) = watch::channel(None); + ( + ShutdownRequest { + requested, + deadline, + }, + requested_rx, + deadline_rx, + ) +} + +#[async_trait] +trait MdnsReceiver: Send { + async fn next(&mut self) -> Result; +} + +#[async_trait] +impl MdnsReceiver for mdns_sd::Receiver { + async fn next(&mut self) -> Result { + self.recv_async() + .await + .map_err(|error| provider_error(format!("mDNS event stream ended: {error}"), true)) + } +} + +#[allow(clippy::too_many_arguments)] +fn start_mdns_task( + config: MdnsDiscoveryConfig, + state: Arc, + own_endpoint: Arc>>, + events: impl MdnsReceiver + 'static, + monitor: impl MdnsReceiver + 'static, + cleanup: DaemonCleanup, + announcements: mpsc::Receiver, + shutdown: watch::Receiver, + shutdown_deadline: watch::Receiver>, +) -> ProviderTask { + let cleanup = Arc::new(tokio::sync::Mutex::new(cleanup)); + let worker_cleanup = cleanup.clone(); + let worker_state = state.clone(); + let worker_endpoint = own_endpoint.clone(); + let worker_shutdown_deadline = shutdown_deadline.clone(); + ProviderTask::spawn( + PROVIDER, + state, + shutdown.clone(), + async move { + let mut cleanup = worker_cleanup.lock().await; + run_mdns_events( + config, + worker_state, + worker_endpoint, + events, + monitor, + &mut *cleanup, + announcements, + worker_shutdown_deadline, + ) + .await + }, + move || async move { + // The worker has been joined, including after panic. Its async lock + // guard is gone, while original registration keys remain owned here. + let deadline = shutdown_deadline + .borrow() + .unwrap_or_else(|| deadline_after(SHUTDOWN_BUDGET)); + let result = shutdown_daemon_until(&mut *cleanup.lock().await, deadline).await; + if *shutdown.borrow() || shutdown.has_changed().is_err() { + own_endpoint + .lock() + .unwrap_or_else(|error| error.into_inner()) + .take(); + } + // DaemonCleanup's fallback must also finish before restart admission. + drop(cleanup); + result + }, + ) +} + +#[allow(clippy::too_many_arguments)] +async fn run_mdns_events( + config: MdnsDiscoveryConfig, + state: Arc, + own_endpoint: Arc>>, + mut events: impl MdnsReceiver, + mut monitor: impl MdnsReceiver, + cleanup: &mut DaemonCleanup, + mut announcements: mpsc::Receiver, + mut shutdown: watch::Receiver>, +) -> Result<(), DiscoveryError> { + let mut instances = HashMap::::new(); + let mut order = Vec::::new(); + let mut expected_shutdown = false; + loop { + if shutdown.borrow().is_some() { + expected_shutdown = true; + break; + } + tokio::select! { + changed = shutdown.changed() => { + if changed.is_err() || shutdown.borrow().is_some() { + expected_shutdown = true; + break; + } + } + Some(request) = announcements.recv() => { + if shutdown.borrow().is_some() { + let _ = request.reply.send(Err(provider_error("provider is shut down", false))); + expected_shutdown = true; + break; + } + let result = replace_announcement_with_shutdown( + cleanup, + &config, + request.endpoint, + &mut shutdown, + ).await; + if let Some(current) = &cleanup.owned.current { + *own_endpoint.lock().unwrap_or_else(|error| error.into_inner()) = + Some(current.endpoint.clone()); + } + remove_owned_instances(&cleanup.owned, &mut instances, &mut order); + publish_instances(&state, &instances, &order, config.max_candidates); + let _ = request.reply.send(result); + // A failed retirement must not accumulate registrations on + // subsequent updates. Cleanup still owns both original keys. + expected_shutdown = shutdown.borrow().is_some(); + if expected_shutdown || cleanup.owned.keys.len() > 1 { + break; + } + } + event = events.next() => match event { + Ok(ServiceEvent::ServiceResolved(service)) => { + let fullname = service.get_fullname().to_string(); + let endpoints = bounded_mdns_endpoints( + service.get_addresses().iter().map(|address| address.to_ip_addr()), + service.get_port(), + config.max_candidates, + ); + if cleanup.owned.matches(&fullname, &endpoints) || service.get_property_val_str("cluster") != Some(config.cluster_id.as_str()) { + if remove_instance(&mut instances, &mut order, &fullname) { + publish_instances(&state, &instances, &order, config.max_candidates); + } + continue; + } + store_instance(&mut instances, &mut order, fullname, endpoints, &config); + publish_instances(&state, &instances, &order, config.max_candidates); + } + Ok(ServiceEvent::ServiceRemoved(_, fullname)) => { + if remove_instance(&mut instances, &mut order, &fullname) { + publish_instances(&state, &instances, &order, config.max_candidates); + } + } + Ok(ServiceEvent::SearchStopped(_)) => { + expected_shutdown = shutdown.borrow().is_some(); + if !expected_shutdown { + tracing::warn!(provider = PROVIDER, "mDNS browse stopped unexpectedly"); + } + break; + } + Err(error) => { + tracing::warn!(%error, provider = PROVIDER, "mDNS event stream ended"); + break; + } + Ok(_) => {} + }, + event = monitor.next() => match event { + Ok(DaemonEvent::NameChange(change)) => { + let updated = match cleanup.owned.name_change(&change) { + Ok(updated) => updated, + Err(error) => { + tracing::warn!(%error, "mDNS own-name history exhausted"); + break; + } + }; + if updated { + remove_owned_instances(&cleanup.owned, &mut instances, &mut order); + publish_instances(&state, &instances, &order, config.max_candidates); + tracing::debug!( + original = %change.original, + new_name = %change.new_name, + "mDNS renamed the local service after a conflict" + ); + } + } + Ok(DaemonEvent::Error(error)) => { + tracing::warn!(%error, provider = PROVIDER, "mDNS daemon failed"); + break; + } + Err(error) => { + tracing::warn!(%error, provider = PROVIDER, "mDNS monitor stream ended"); + break; + } + Ok(_) => {} + } + } + } + announcements.close(); + while let Ok(request) = announcements.try_recv() { + let error = if expected_shutdown || shutdown.borrow().is_some() { + provider_error("provider is shut down", false) + } else { + provider_error("mDNS announcement task stopped", true) + }; + let _ = request.reply.send(Err(error)); + } + if expected_shutdown { + Ok(()) + } else { + Err(provider_error("mDNS browse ended unexpectedly", true)) + } +} + +#[cfg(test)] +async fn wait_for_shutdown( + completion: Option>>>, +) -> Result<(), DiscoveryError> { + let Some(mut completion) = completion else { + return Ok(()); + }; + loop { + if let Some(result) = completion.borrow_and_update().clone() { + return result; + } + completion.changed().await.map_err(|_| { + provider_error( + "mDNS cleanup task ended without an acknowledgement result", + false, + ) + })?; + } +} + +#[async_trait] +trait ShutdownDaemon: Send { + async fn unregister(&mut self, deadline: tokio::time::Instant) -> Result<(), DiscoveryError>; + async fn shutdown(&mut self) -> Result<(), DiscoveryError>; +} + +#[async_trait] +trait RegistrationDaemon: Send { + fn register(&mut self, service: ServiceInfo) -> Result<(), DiscoveryError>; + async fn withdraw(&mut self, key: &str) -> Result<(), DiscoveryError>; + async fn terminate(&mut self) -> Result<(), DiscoveryError>; + fn fallback(&mut self, keys: &BTreeSet); +} + +struct LiveDaemon { + daemon: ServiceDaemon, + service_type: String, +} + +struct DaemonCleanup { + daemon: D, + owned: OwnedAnnouncements, + finished: bool, +} + +#[async_trait] +impl ShutdownDaemon for DaemonCleanup { + async fn unregister(&mut self, deadline: tokio::time::Instant) -> Result<(), DiscoveryError> { + let mut result = Ok(()); + let keys = self.owned.keys.clone(); + for (index, key) in keys.iter().enumerate() { + let remaining_keys = (keys.len() - index) as u32; + let now = tokio::time::Instant::now(); + let key_deadline = now + .checked_add(deadline.saturating_duration_since(now) / remaining_keys) + .unwrap_or(deadline) + .min(deadline); + let withdrawal = tokio::time::timeout_at(key_deadline, self.daemon.withdraw(key)) + .await + .unwrap_or_else(|_| { + Err(provider_error( + "mDNS unregister acknowledgement timed out", + false, + )) + }); + match withdrawal { + Ok(()) => { + self.owned.keys.remove(key); + } + Err(error) => { + result = result.and(Err(error)); + } + } + } + result + } + + async fn shutdown(&mut self) -> Result<(), DiscoveryError> { + self.daemon.terminate().await?; + self.owned = OwnedAnnouncements::default(); + self.finished = true; + Ok(()) + } +} + +#[async_trait] +impl RegistrationDaemon for LiveDaemon { + fn register(&mut self, service: ServiceInfo) -> Result<(), DiscoveryError> { + self.daemon + .register(service) + .map_err(|error| provider_error(format!("cannot register mDNS service: {error}"), true)) + } + + async fn withdraw(&mut self, key: &str) -> Result<(), DiscoveryError> { + let ack = enqueue_daemon_command(|| self.daemon.unregister(key)).await?; + // OK and NotFound both mean this original registration key is gone. + ack.recv_async().await.map_err(|error| { + provider_error( + format!("mDNS unregister acknowledgement failed: {error}"), + false, + ) + })?; + Ok(()) + } + + async fn terminate(&mut self) -> Result<(), DiscoveryError> { + let _ = self.daemon.stop_browse(&self.service_type); + let ack = enqueue_daemon_command(|| self.daemon.shutdown()).await?; + let status = ack.recv_async().await.map_err(|error| { + provider_error( + format!("mDNS shutdown acknowledgement failed: {error}"), + false, + ) + })?; + if status != DaemonStatus::Shutdown { + return Err(provider_error( + "unexpected mDNS shutdown acknowledgement", + false, + )); + } + Ok(()) + } + + fn fallback(&mut self, keys: &BTreeSet) { + for key in keys { + let _ = self.daemon.unregister(key); + } + let _ = self.daemon.stop_browse(&self.service_type); + let _ = self.daemon.shutdown(); + } +} + +async fn enqueue_daemon_command( + mut send: impl FnMut() -> mdns_sd::Result, +) -> Result { + loop { + match send() { + Ok(result) => return Ok(result), + // The enclosing ACK deadline also bounds command-queue retries. + Err(mdns_sd::Error::Again) => tokio::time::sleep(Duration::from_millis(10)).await, + Err(error) => { + return Err(provider_error( + format!("mDNS command failed: {error}"), + false, + )); + } + } + } +} + +impl Drop for DaemonCleanup { + fn drop(&mut self) { + if !self.finished { + // Runtime teardown/panic fallback only; normal shutdown has one + // owner and awaits ACKs. UDP delivery to every LAN peer is not guaranteed. + self.daemon.fallback(&self.owned.keys); + } + } +} + +async fn shutdown_daemon_until( + daemon: &mut impl ShutdownDaemon, + deadline: tokio::time::Instant, +) -> Result<(), DiscoveryError> { + let now = tokio::time::Instant::now(); + // Reserve half the common deadline for daemon termination, even when + // withdrawal errors or its ACK never arrives. + let withdrawal_deadline = now + .checked_add(deadline.saturating_duration_since(now) / 2) + .unwrap_or(deadline) + .min(deadline); + let withdrawal = + tokio::time::timeout_at(withdrawal_deadline, daemon.unregister(withdrawal_deadline)) + .await + .unwrap_or_else(|_| { + Err(provider_error( + "mDNS unregister acknowledgement timed out", + false, + )) + }); + let shutdown = daemon.shutdown(); + tokio::pin!(shutdown); + let shutdown = tokio::select! { + biased; + result = &mut shutdown => result, + () = tokio::time::sleep_until(deadline) => Err(provider_error( + "mDNS shutdown acknowledgement timed out", + false, + )), + }; + withdrawal.and(shutdown) +} + +#[cfg(test)] +async fn shutdown_daemon( + daemon: &mut impl ShutdownDaemon, + budget: Duration, +) -> Result<(), DiscoveryError> { + shutdown_daemon_until(daemon, deadline_after(budget)).await +} + +struct InstanceView { + endpoints: Box<[String]>, + observed_at: StdInstant, +} + +fn store_instance( + instances: &mut HashMap, + order: &mut Vec, + fullname: String, + mut endpoints: Vec, + config: &MdnsDiscoveryConfig, +) { + if !instances.contains_key(&fullname) && instances.len() >= config.max_instances { + return; + } + // Duplicate contributions also consume capacity. A replacement reclaims + // its own old allocation before admission; overflow is not retained off-view. + let used: usize = instances + .iter() + .filter(|(name, _)| *name != &fullname) + .map(|(_, view)| view.endpoints.len()) + .sum(); + endpoints.truncate(config.max_candidates.saturating_sub(used)); + if endpoints.is_empty() { + remove_instance(instances, order, &fullname); + return; + } + if !instances.contains_key(&fullname) { + order.push(fullname.clone()); + } + instances.insert( + fullname, + InstanceView { + // Truncating a Vec alone retains its original capacity per instance. + // Boxed storage also releases that otherwise multiplicative slack. + endpoints: endpoints.into_boxed_slice(), + observed_at: StdInstant::now(), + }, + ); +} + +fn publish_instances( + state: &DynamicState, + instances: &HashMap, + order: &[String], + max_candidates: usize, +) { + let peers = flatten_instances(instances, order, max_candidates); + state.observe_at( + peers + .into_iter() + .filter_map(|peer| { + let at = instances + .values() + .filter(|view| view.endpoints.contains(&peer)) + .map(|view| view.observed_at) + .max()?; + Some((peer, at)) + }) + .collect(), + ); +} + +struct CurrentAnnouncement { + key: String, + endpoint: String, +} + +#[derive(Default)] +struct OwnedAnnouncements { + current: Option, + // mdns-sd 0.21.3 register_service/remove_entry use the original lowercase + // ServiceInfo fullname. NameChange only updates per-interface wire aliases; + // unregister_service resolves those aliases when constructing goodbyes. + keys: BTreeSet, + // Keep retired names/endpoints until daemon termination: browse and monitor + // streams are independent, and cached/queued resolutions can arrive late. + names: BTreeSet, + endpoints: BTreeSet, + generation: usize, +} + +impl OwnedAnnouncements { + fn accept(&mut self, fullname: String, endpoint: String) { + let key = fullname.to_lowercase(); + self.keys.insert(key.clone()); + self.names.insert(key.clone()); + self.endpoints.insert(endpoint.clone()); + self.current = Some(CurrentAnnouncement { key, endpoint }); + self.generation += 1; + } + + fn matches(&self, fullname: &str, endpoints: &[String]) -> bool { + self.names.contains(&fullname.to_lowercase()) + || endpoints + .iter() + .any(|endpoint| self.endpoints.contains(endpoint)) + } + + fn name_change(&mut self, change: &DnsNameChange) -> Result { + if change.rr_type != RRType::SRV || !self.names.contains(&change.original.to_lowercase()) { + return Ok(false); + } + let name = change.new_name.to_lowercase(); + if !self.names.contains(&name) && self.names.len() >= MAX_OWN_HISTORY { + return Err(provider_error("mDNS own-name history limit reached", true)); + } + self.names.insert(name); + Ok(true) + } +} + +#[cfg(test)] +async fn replace_announcement( + cleanup: &mut DaemonCleanup, + config: &MdnsDiscoveryConfig, + endpoint: String, +) -> Result<(), DiscoveryError> { + let (_shutdown, mut shutdown) = watch::channel(None); + replace_announcement_with_shutdown(cleanup, config, endpoint, &mut shutdown).await +} + +async fn replace_announcement_with_shutdown( + cleanup: &mut DaemonCleanup, + config: &MdnsDiscoveryConfig, + endpoint: String, + shutdown: &mut watch::Receiver>, +) -> Result<(), DiscoveryError> { + if cleanup.owned.names.len() >= MAX_OWN_HISTORY + || cleanup.owned.endpoints.len() >= MAX_OWN_HISTORY + || cleanup.owned.keys.len() > 1 + { + return Err(provider_error( + "mDNS announcement history limit reached", + true, + )); + } + let mut config = config.clone(); + if cleanup.owned.current.is_some() { + // A distinct ORIGINAL key lets us register first (failure leaves the + // old service intact), then withdraw its old ServiceInfo/endpoint. + // Reusing the key would overwrite that info before its goodbye; using + // an observed alias would unregister NotFound instead of the service. + config.instance_name = format!( + "nx-{}-{}", + &blake3::hash(config.instance_name.as_bytes()).to_hex()[..16], + cleanup.owned.generation, + ); + } + let (service, fullname) = build_service( + &config, + &cluster_service_type(&config.cluster_id), + &endpoint, + )?; + if cleanup.owned.names.contains(&fullname.to_lowercase()) { + return Err(provider_error( + "mDNS replacement key is already owned", + true, + )); + } + let previous = cleanup + .owned + .current + .as_ref() + .map(|current| current.key.clone()); + cleanup.daemon.register(service)?; + cleanup.owned.accept(fullname, endpoint); + if let Some(previous) = previous { + let withdrawal = cleanup.daemon.withdraw(&previous); + tokio::pin!(withdrawal); + let timeout = tokio::time::sleep(SHUTDOWN_BUDGET / 2); + tokio::pin!(timeout); + let result = tokio::select! { + biased; + changed = shutdown.changed() => { + if changed.is_err() || shutdown.borrow().is_some() { + Err(provider_error("provider is shut down", false)) + } else { + Err(provider_error("mDNS announcement task stopped", true)) + } + } + result = &mut withdrawal => result, + () = &mut timeout => { + Err(provider_error("mDNS replacement withdrawal timed out", true)) + } + }; + result?; + cleanup.owned.keys.remove(&previous); + } + Ok(()) +} + +fn remove_owned_instances( + owned: &OwnedAnnouncements, + instances: &mut HashMap, + order: &mut Vec, +) { + instances.retain(|name, view| !owned.matches(name, &view.endpoints)); + order.retain(|name| instances.contains_key(name)); +} + +fn remove_instance( + instances: &mut HashMap, + order: &mut Vec, + fullname: &str, +) -> bool { + let removed = instances.remove(fullname).is_some(); + if removed { + order.retain(|known| known != fullname); + } + removed +} + +fn flatten_instances( + instances: &HashMap, + order: &[String], + max_candidates: usize, +) -> Vec { + let mut peers = Vec::new(); + for fullname in order { + let Some(view) = instances.get(fullname) else { + continue; + }; + for endpoint in &view.endpoints { + if peers.len() == max_candidates { + return peers; + } + if !peers.contains(endpoint) { + peers.push(endpoint.clone()); + } + } + } + peers +} + +fn dialable_mdns_address(address: IpAddr, port: u16) -> Option { + if port == 0 || address.is_unspecified() || address.is_multicast() { + return None; + } + if matches!(address, IpAddr::V6(address) if address.is_unicast_link_local()) { + return None; + } + Some(SocketAddr::new(address, port).to_string()) +} + +fn bounded_mdns_endpoints( + addresses: impl IntoIterator, + port: u16, + max_candidates: usize, +) -> Vec { + let mut endpoints = BTreeSet::new(); + for address in addresses { + if let Some(endpoint) = dialable_mdns_address(address, port) { + endpoints.insert(endpoint); + if endpoints.len() > max_candidates { + endpoints.pop_last(); + } + } + } + endpoints.into_iter().collect() +} + +fn cluster_service_type(cluster_id: &str) -> String { + let hash = blake3::hash(cluster_id.as_bytes()).to_hex(); + format!("_c{}._sub.{SERVICE_BASE}", &hash[..16]) +} + +fn split_endpoint(endpoint: &str) -> Result<(String, u16), DiscoveryError> { + if let Ok(socket) = endpoint.parse::() { + return Ok((socket.ip().to_string(), socket.port())); + } + let (host, port) = endpoint + .rsplit_once(':') + .ok_or_else(|| provider_error("advertised endpoint must include a port", false))?; + let port = port + .parse::() + .map_err(|_| provider_error("advertised endpoint has an invalid port", false))?; + Ok((host.to_string(), port)) +} + +fn build_service( + config: &MdnsDiscoveryConfig, + service_type: &str, + endpoint: &str, +) -> Result<(ServiceInfo, String), DiscoveryError> { + let (host, port) = split_endpoint(endpoint)?; + let hostname = format!( + "numax-{}.local.", + &blake3::hash(config.instance_name.as_bytes()).to_hex()[..16] + ); + let properties = &[("cluster", config.cluster_id.as_str())]; + let service = match host.parse::() { + Ok(ip) => ServiceInfo::new( + service_type, + &config.instance_name, + &hostname, + ip, + port, + properties.as_slice(), + ), + Err(_) if host.ends_with(".local") => ServiceInfo::new( + service_type, + &config.instance_name, + &format!("{host}."), + "", + port, + properties.as_slice(), + ) + .map(ServiceInfo::enable_addr_auto), + Err(_) => { + return Err(provider_error( + "mDNS announcements require an IP address or .local hostname", + false, + )); + } + } + .map_err(|error| provider_error(format!("invalid mDNS service: {error}"), false))?; + let fullname = service.get_fullname().to_string(); + Ok((service, fullname)) +} + +fn validate_config(config: &MdnsDiscoveryConfig) -> Result<(), DiscoveryError> { + validate_event_capacity(PROVIDER, config.event_capacity)?; + if config.instance_name.is_empty() || config.instance_name.len() > 63 { + return Err(invalid("instance_name length must be in 1..=63 bytes")); + } + if config.instance_name.chars().any(char::is_control) { + return Err(invalid("instance_name must not contain control characters")); + } + if config.cluster_id.is_empty() || config.cluster_id.len() > 128 { + return Err(invalid("cluster_id length must be in 1..=128 bytes")); + } + if config.max_instances == 0 || config.max_candidates == 0 { + return Err(invalid("limits must be greater than zero")); + } + Ok(()) +} + +fn invalid(message: impl Into) -> DiscoveryError { + DiscoveryError::InvalidConfiguration { + provider: PROVIDER.to_string(), + message: message.into(), + } +} + +fn provider_error(message: impl Into, retryable: bool) -> DiscoveryError { + DiscoveryError::Provider { + provider: PROVIDER.to_string(), + message: message.into(), + retryable, + } +} + +#[cfg(test)] +mod tests { + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + use super::*; + + fn instance(endpoints: Vec) -> InstanceView { + InstanceView { + endpoints: endpoints.into_boxed_slice(), + observed_at: StdInstant::now(), + } + } + + #[test] + fn global_endpoint_budget_counts_duplicates_and_reclaims_removals_and_replacements() { + let mut config = MdnsDiscoveryConfig::new("test"); + config.max_candidates = 32; + config.max_instances = 1024; + let mut instances = HashMap::new(); + let mut order = Vec::new(); + let endpoints: Vec<_> = (1..=16).map(|port| format!("127.0.0.1:{port}")).collect(); + for index in 0..1024 { + store_instance( + &mut instances, + &mut order, + format!("peer-{index}"), + endpoints.clone(), + &config, + ); + assert!( + instances + .values() + .map(|view| view.endpoints.len()) + .sum::() + <= config.max_candidates + ); + } + assert_eq!(order, ["peer-0", "peer-1"]); + assert_eq!(flatten_instances(&instances, &order, 32), endpoints); + store_instance( + &mut instances, + &mut order, + "peer-0".into(), + vec!["127.0.0.1:99".into()], + &config, + ); + store_instance( + &mut instances, + &mut order, + "replacement".into(), + endpoints.clone(), + &config, + ); + assert_eq!( + instances["replacement"].endpoints.as_ref(), + &endpoints[..15] + ); + assert_eq!( + instances + .values() + .map(|view| view.endpoints.len()) + .sum::(), + 32 + ); + assert!(remove_instance(&mut instances, &mut order, "peer-1")); + store_instance( + &mut instances, + &mut order, + "after-removal".into(), + endpoints.clone(), + &config, + ); + assert_eq!( + instances["after-removal"].endpoints.as_ref(), + endpoints.as_slice() + ); + assert_eq!(order, ["peer-0", "replacement", "after-removal"]); + assert_eq!( + instances + .values() + .map(|view| view.endpoints.len()) + .sum::(), + 32 + ); + } + + #[test] + fn removing_one_instance_does_not_renew_other_instances() { + let config = MdnsDiscoveryConfig::new("test"); + let mut instances = HashMap::new(); + let mut order = Vec::new(); + store_instance( + &mut instances, + &mut order, + "a".into(), + vec!["a:1".into()], + &config, + ); + store_instance( + &mut instances, + &mut order, + "b".into(), + vec!["b:2".into()], + &config, + ); + let observed = instances["a"].observed_at; + let state = DynamicState::new(8); + publish_instances(&state, &instances, &order, 8); + remove_instance(&mut instances, &mut order, "b"); + publish_instances(&state, &instances, &order, 8); + assert_eq!(state.snapshot().observations().unwrap(), [observed]); + } + + struct ControlledDaemon { + calls: tokio::sync::mpsc::Sender<&'static str>, + unregister_ack: Option>>, + shutdown_ack: Option>>, + } + + #[async_trait] + impl ShutdownDaemon for ControlledDaemon { + async fn unregister( + &mut self, + _deadline: tokio::time::Instant, + ) -> Result<(), DiscoveryError> { + self.calls.send("unregister").await.unwrap(); + self.unregister_ack + .take() + .unwrap() + .await + .map_err(|_| provider_error("unregister ack channel closed", false))? + } + async fn shutdown(&mut self) -> Result<(), DiscoveryError> { + self.calls.send("shutdown").await.unwrap(); + self.shutdown_ack + .take() + .unwrap() + .await + .map_err(|_| provider_error("shutdown ack channel closed", false))? + } + } + + #[tokio::test] + async fn shutdown_awaits_both_acks_and_waiter_cancellation_preserves_the_single_owner() { + let (calls, mut call_rx) = tokio::sync::mpsc::channel(2); + let (unregister_tx, unregister_ack) = tokio::sync::oneshot::channel(); + let (shutdown_tx, shutdown_ack) = tokio::sync::oneshot::channel(); + let mut daemon = ControlledDaemon { + calls, + unregister_ack: Some(unregister_ack), + shutdown_ack: Some(shutdown_ack), + }; + let (complete_tx, complete_rx) = watch::channel(None); + let owner = tokio::spawn(async move { + complete_tx.send_replace(Some( + shutdown_daemon(&mut daemon, Duration::from_secs(2)).await, + )); + }); + let waiter_rx = complete_rx.clone(); + let waiter = tokio::spawn(wait_for_shutdown(Some(waiter_rx))); + assert_eq!(call_rx.recv().await, Some("unregister")); + assert!(call_rx.try_recv().is_err()); + assert!(complete_rx.borrow().is_none()); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + unregister_tx.send(Ok(())).unwrap(); + assert_eq!(call_rx.recv().await, Some("shutdown")); + assert!(complete_rx.borrow().is_none()); + shutdown_tx.send(Ok(())).unwrap(); + wait_for_shutdown(Some(complete_rx.clone())).await.unwrap(); + wait_for_shutdown(Some(complete_rx)).await.unwrap(); + owner.await.unwrap(); + assert!(call_rx.recv().await.is_none()); + } + + #[tokio::test] + async fn dropping_provider_signals_cleanup_without_aborting_acknowledgements() { + let provider = MdnsDiscovery::new(MdnsDiscoveryConfig::new("drop-test")).unwrap(); + let (calls, mut call_rx) = tokio::sync::mpsc::channel(2); + let (unregister_tx, unregister_ack) = tokio::sync::oneshot::channel(); + let (shutdown_tx, shutdown_ack) = tokio::sync::oneshot::channel(); + let mut daemon = ControlledDaemon { + calls, + unregister_ack: Some(unregister_ack), + shutdown_ack: Some(shutdown_ack), + }; + let (shutdown, mut cancel_rx, _deadline_rx) = shutdown_channels(); + let task = ProviderTask::spawn( + PROVIDER, + provider.inner.state.clone(), + cancel_rx.clone(), + async move { + cancel_rx.changed().await.unwrap(); + assert!(*cancel_rx.borrow()); + Ok(()) + }, + move || async move { shutdown_daemon(&mut daemon, Duration::from_secs(2)).await }, + ); + let completion = task.clone(); + { + let mut lifecycle = provider.inner.lifecycle.lock().unwrap(); + lifecycle.shutdown = Some(shutdown); + lifecycle.task = Some(task); + } + drop(provider); + tokio::time::timeout(Duration::from_secs(2), async { + assert_eq!(call_rx.recv().await, Some("unregister")); + unregister_tx.send(Ok(())).unwrap(); + assert_eq!(call_rx.recv().await, Some("shutdown")); + shutdown_tx.send(Ok(())).unwrap(); + completion.join().await.unwrap(); + }) + .await + .unwrap(); + } + + #[tokio::test] + async fn withdrawal_error_still_awaits_daemon_shutdown_and_reports_error() { + let (calls, mut call_rx) = tokio::sync::mpsc::channel(2); + let (unregister_tx, unregister_ack) = tokio::sync::oneshot::channel(); + let (shutdown_tx, shutdown_ack) = tokio::sync::oneshot::channel(); + let mut daemon = ControlledDaemon { + calls, + unregister_ack: Some(unregister_ack), + shutdown_ack: Some(shutdown_ack), + }; + let task = + tokio::spawn(async move { shutdown_daemon(&mut daemon, Duration::from_secs(2)).await }); + assert_eq!(call_rx.recv().await, Some("unregister")); + drop(unregister_tx); + assert_eq!(call_rx.recv().await, Some("shutdown")); + assert!(!task.is_finished()); + shutdown_tx.send(Ok(())).unwrap(); + assert_eq!( + task.await.unwrap(), + Err(provider_error("unregister ack channel closed", false)) + ); + } + + #[tokio::test] + async fn missing_ack_deadlines_bound_withdrawal_and_shutdown() { + let (calls, mut call_rx) = tokio::sync::mpsc::channel(2); + let (_unregister_tx, unregister_ack) = tokio::sync::oneshot::channel(); + let (_shutdown_tx, shutdown_ack) = tokio::sync::oneshot::channel(); + let mut daemon = ControlledDaemon { + calls, + unregister_ack: Some(unregister_ack), + shutdown_ack: Some(shutdown_ack), + }; + let task = + tokio::spawn( + async move { shutdown_daemon(&mut daemon, Duration::from_millis(20)).await }, + ); + let result = tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap(); + assert_eq!(call_rx.recv().await, Some("unregister")); + assert_eq!(call_rx.recv().await, Some("shutdown")); + assert_eq!( + result, + Err(provider_error( + "mDNS unregister acknowledgement timed out", + false + )) + ); + } + + #[tokio::test] + async fn daemon_shutdown_ack_error_is_not_reported_as_success() { + let (calls, _call_rx) = tokio::sync::mpsc::channel(2); + let (unregister_tx, unregister_ack) = tokio::sync::oneshot::channel(); + let (shutdown_tx, shutdown_ack) = tokio::sync::oneshot::channel(); + unregister_tx.send(Ok(())).unwrap(); + drop(shutdown_tx); + let mut daemon = ControlledDaemon { + calls, + unregister_ack: Some(unregister_ack), + shutdown_ack: Some(shutdown_ack), + }; + assert_eq!( + shutdown_daemon(&mut daemon, Duration::from_secs(1)).await, + Err(provider_error("shutdown ack channel closed", false)) + ); + } + + #[tokio::test] + async fn full_daemon_queue_is_retried_but_permanent_errors_are_not() { + let mut attempts = 0; + let value = enqueue_daemon_command(|| { + attempts += 1; + if attempts == 1 { + Err(mdns_sd::Error::Again) + } else { + Ok(42) + } + }) + .await + .unwrap(); + assert_eq!(value, 42); + assert_eq!(attempts, 2); + assert!( + enqueue_daemon_command::<()>(|| Err(mdns_sd::Error::DaemonShutdown)) + .await + .is_err() + ); + } + + #[test] + fn cluster_service_types_are_stable_and_isolated() { + assert_eq!(cluster_service_type("a"), cluster_service_type("a")); + assert_ne!(cluster_service_type("a"), cluster_service_type("b")); + assert!(cluster_service_type("a").ends_with(SERVICE_BASE)); + } + + #[test] + fn reducer_deduplicates_shared_endpoints_and_preserves_instance_order() { + let instances = HashMap::from([ + ("a".to_string(), instance(vec!["127.0.0.1:1".to_string()])), + ( + "b".to_string(), + instance(vec!["127.0.0.1:1".to_string(), "127.0.0.1:2".to_string()]), + ), + ]); + assert_eq!( + flatten_instances(&instances, &["a".into(), "b".into()], 8), + ["127.0.0.1:1", "127.0.0.1:2"] + ); + } + + #[test] + fn undialable_addresses_are_filtered() { + assert!(dialable_mdns_address("0.0.0.0".parse().unwrap(), 9000).is_none()); + assert!(dialable_mdns_address("ff02::1".parse().unwrap(), 9000).is_none()); + assert!(dialable_mdns_address("fe80::1".parse().unwrap(), 9000).is_none()); + assert_eq!( + dialable_mdns_address("127.0.0.1".parse().unwrap(), 9000), + Some("127.0.0.1:9000".into()) + ); + } + + #[test] + fn resolved_instance_addresses_are_bounded_and_deterministic() { + let addresses = [ + "127.0.0.3".parse().unwrap(), + "127.0.0.1".parse().unwrap(), + "127.0.0.2".parse().unwrap(), + "127.0.0.1".parse().unwrap(), + ]; + + assert_eq!( + bounded_mdns_endpoints(addresses, 9000, 2), + ["127.0.0.1:9000", "127.0.0.2:9000"] + ); + } + + #[test] + fn rejected_resolution_removes_a_previously_accepted_instance() { + let mut instances = HashMap::from([( + "peer._numax._tcp.local.".into(), + instance(vec!["127.0.0.1:9000".into()]), + )]); + let mut order = vec!["peer._numax._tcp.local.".into()]; + + assert!(remove_instance( + &mut instances, + &mut order, + "peer._numax._tcp.local." + )); + assert!(instances.is_empty()); + assert!(order.is_empty()); + } + + #[test] + fn service_name_conflicts_update_the_self_filter() { + let mut owned = OwnedAnnouncements::default(); + owned.accept("node._numax._tcp.local.".into(), "127.0.0.1:9000".into()); + let change = DnsNameChange { + original: "node._numax._tcp.local.".into(), + new_name: "node (2)._numax._tcp.local.".into(), + rr_type: RRType::SRV, + intf_name: "test".into(), + }; + + assert!(owned.name_change(&change).unwrap()); + assert!(owned.matches(&change.original.to_uppercase(), &[])); + assert!(owned.matches(&change.new_name.to_uppercase(), &[])); + assert_eq!(owned.keys, BTreeSet::from([change.original.clone()])); + let mut other_interface = change.clone(); + other_interface.new_name = "node (3)._numax._tcp.local.".into(); + assert!(owned.name_change(&other_interface).unwrap()); + assert!(owned.matches(&change.new_name, &[])); + assert!(owned.matches(&other_interface.new_name, &[])); + } + + #[derive(Default)] + struct FakeRegistrations { + active: HashMap, + calls: Vec, + fail_register: bool, + fail_withdraw: bool, + } + + struct FakeDaemon { + state: Arc>, + withdrawal: Option<(oneshot::Sender, oneshot::Receiver<()>)>, + missing_withdraw_acks: bool, + termination: Option<(oneshot::Sender<()>, oneshot::Receiver<()>)>, + } + + #[async_trait] + impl RegistrationDaemon for FakeDaemon { + fn register(&mut self, service: ServiceInfo) -> Result<(), DiscoveryError> { + let mut state = self.state.lock().unwrap(); + if state.fail_register { + return Err(provider_error("injected register failure", true)); + } + let key = service.get_fullname().to_lowercase(); + state.calls.push(format!("register:{key}")); + state.active.insert(key, service); + Ok(()) + } + + async fn withdraw(&mut self, key: &str) -> Result<(), DiscoveryError> { + if let Some((started, ack)) = self.withdrawal.take() { + started.send(key.to_string()).unwrap(); + ack.await + .map_err(|_| provider_error("injected missing ACK", false))?; + } + { + let mut state = self.state.lock().unwrap(); + state.calls.push(format!("unregister:{key}")); + if state.fail_withdraw { + return Err(provider_error("injected withdrawal failure", false)); + } + } + if self.missing_withdraw_acks { + std::future::pending::<()>().await; + } + let mut state = self.state.lock().unwrap(); + // Unlike a wire alias, only the original key removes the record. + state.active.remove(key); + Ok(()) + } + + async fn terminate(&mut self) -> Result<(), DiscoveryError> { + self.state.lock().unwrap().calls.push("shutdown".into()); + if let Some((started, ack)) = self.termination.take() { + started.send(()).unwrap(); + ack.await + .map_err(|_| provider_error("injected missing shutdown ACK", false))?; + } + // A confirmed daemon termination retires every registration, even + // if an individual unregister ACK was lost. + self.state.lock().unwrap().active.clear(); + Ok(()) + } + + fn fallback(&mut self, keys: &BTreeSet) { + let mut state = self.state.lock().unwrap(); + for key in keys { + state.calls.push(format!("fallback:{key}")); + state.active.remove(key); + } + } + } + + fn fake_cleanup() -> DaemonCleanup { + DaemonCleanup { + daemon: FakeDaemon { + state: Arc::new(StdMutex::new(FakeRegistrations::default())), + withdrawal: None, + missing_withdraw_acks: false, + termination: None, + }, + owned: OwnedAnnouncements::default(), + finished: false, + } + } + + struct FakeEvents { + receiver: mpsc::Receiver<(T, oneshot::Sender<()>)>, + processed: Option>, + } + + #[async_trait] + impl MdnsReceiver for FakeEvents { + async fn next(&mut self) -> Result { + // The next poll acknowledges that the previous event's handler + // completed, not merely that its input was dequeued. + if let Some(processed) = self.processed.take() { + let _ = processed.send(()); + } + let (event, processed) = self + .receiver + .recv() + .await + .ok_or_else(|| provider_error("fake stream closed", false))?; + self.processed = Some(processed); + Ok(event) + } + } + + async fn deliver(sender: &mpsc::Sender<(T, oneshot::Sender<()>)>, event: T) { + let (processed, ack) = oneshot::channel(); + assert!(sender.send((event, processed)).await.is_ok()); + tokio::time::timeout(Duration::from_secs(2), ack) + .await + .unwrap() + .unwrap(); + } + + type EventSender = mpsc::Sender<(T, oneshot::Sender<()>)>; + + fn fake_generation( + provider: &MdnsDiscovery, + cleanup: DaemonCleanup, + ) -> ( + MdnsGeneration, + EventSender, + EventSender, + ) { + let (events, event_rx) = mpsc::channel(8); + let (monitor, monitor_rx) = mpsc::channel(8); + let (announcements, announcement_rx) = mpsc::channel(8); + let (shutdown, shutdown_rx, shutdown_deadline_rx) = shutdown_channels(); + let task = start_mdns_task( + provider.inner.config.clone(), + provider.inner.state.clone(), + provider.inner.own_endpoint.clone(), + FakeEvents { + receiver: event_rx, + processed: None, + }, + FakeEvents { + receiver: monitor_rx, + processed: None, + }, + cleanup, + announcement_rx, + shutdown_rx, + shutdown_deadline_rx, + ); + ( + MdnsGeneration { + task, + shutdown, + announcements, + }, + events, + monitor, + ) + } + + async fn assert_mdns_finalization_blocks_restart(panic: bool) { + let config = MdnsDiscoveryConfig::new("supervised"); + let provider = MdnsDiscovery::new(config.clone()).unwrap(); + let mut cleanup = fake_cleanup(); + replace_announcement(&mut cleanup, &config, "127.0.0.1:9000".into()) + .await + .unwrap(); + let registrations = cleanup.daemon.state.clone(); + let (entered, termination) = oneshot::channel(); + let (release, released) = oneshot::channel(); + cleanup.daemon.termination = Some((entered, released)); + let (generation, events, _monitor) = fake_generation(&provider, cleanup); + let old = generation.task.clone(); + provider.ensure_started_with(|| Ok(generation)).unwrap(); + let mut observed = provider.watch().await.unwrap(); + let mut foreign = config.clone(); + foreign.instance_name = "foreign".into(); + let (service, _) = + build_service(&foreign, &provider.inner.service_type, "127.0.0.2:9000").unwrap(); + let service = service.as_resolved_service(); + deliver( + &events, + ServiceEvent::ServiceResolved(Box::new(service.clone())), + ) + .await; + assert_eq!( + super::super::next_changed_peers(&mut observed, &[]).await, + ["127.0.0.2:9000"] + ); + assert_eq!(provider.inner.state.snapshot().peers(), ["127.0.0.2:9000"]); + let event = if panic { + provider.inner.state.panic_on_next_observation(); + ServiceEvent::ServiceResolved(Box::new(service.clone())) + } else { + ServiceEvent::SearchStopped(provider.inner.service_type.clone()) + }; + let (processed, _ack) = oneshot::channel(); + events.send((event, processed)).await.unwrap(); + super::super::dynamic::assert_invalidated(&mut observed).await; + termination.await.unwrap(); + assert!(provider.inner.state.snapshot().peers().is_empty()); + // Longer than the coordinator's first 500ms retry. A resubscription + // must keep failing instead of attaching to the dying generation. + assert!( + tokio::time::timeout(Duration::from_millis(600), old.clone().join()) + .await + .is_err() + ); + assert!( + provider + .ensure_started_with(|| panic!("cleanup is still running")) + .is_err() + ); + let (first, second) = tokio::join!(provider.watch(), provider.watch()); + assert!(matches!(first, Err(DiscoveryError::WatchClosed))); + assert!(matches!(second, Err(DiscoveryError::WatchClosed))); + assert!(!old.completion_ready()); + release.send(()).unwrap(); + let error = old.clone().join().await.unwrap_err(); + if panic { + assert!( + matches!(error, DiscoveryError::Provider { message, retryable: false, .. } + if message.contains("provider task failed") && message.contains("panic")) + ); + } else { + assert!(matches!( + error, + DiscoveryError::Provider { + retryable: true, + .. + } + )); + } + assert!(registrations.lock().unwrap().active.is_empty()); + assert_eq!( + registrations.lock().unwrap().calls.last().unwrap(), + "shutdown" + ); + let starts = std::sync::atomic::AtomicUsize::new(0); + let retained = StdMutex::new(Vec::new()); + let start = || { + // Admission is serialized with shutdown and only follows cleanup. + starts.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + assert!(registrations.lock().unwrap().active.is_empty()); + let (generation, events, monitor) = fake_generation(&provider, fake_cleanup()); + retained.lock().unwrap().push((events, monitor)); + Ok(generation) + }; + let (first, second) = tokio::join!(async { provider.ensure_started_with(start) }, async { + provider.ensure_started_with(start) + },); + first.unwrap(); + second.unwrap(); + assert_eq!(starts.load(std::sync::atomic::Ordering::SeqCst), 1); + assert!( + !old.same_generation( + provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .as_ref() + .unwrap() + ) + ); + let mut fresh = provider.watch().await.unwrap(); + assert!(fresh.snapshot().peers().is_empty()); + let sender = retained.lock().unwrap()[0].0.clone(); + deliver(&sender, ServiceEvent::ServiceResolved(Box::new(service))).await; + assert_eq!( + super::super::next_changed_peers(&mut fresh, &[]).await, + ["127.0.0.2:9000"] + ); + provider.shutdown().await.unwrap(); + provider.shutdown().await.unwrap(); + assert!(provider.inner.state.snapshot().peers().is_empty()); + assert!(provider.inner.own_endpoint.lock().unwrap().is_none()); + assert!( + provider + .ensure_started_with(|| panic!("shutdown is terminal")) + .is_err() + ); + } + + #[tokio::test] + async fn shutdown_after_unexpected_finalization_clears_preserved_announcement() { + let provider = MdnsDiscovery::new(MdnsDiscoveryConfig::new("late-shutdown")).unwrap(); + let cleanup = fake_cleanup(); + let registrations = cleanup.daemon.state.clone(); + let (generation, events, _monitor) = fake_generation(&provider, cleanup); + let task = generation.task.clone(); + provider.ensure_started_with(|| Ok(generation)).unwrap(); + provider + .announce(&PeerAnnouncement { + endpoint: "127.0.0.1:9000".into(), + }) + .await + .unwrap(); + drop(events); + assert!(task.join().await.is_err()); + assert!(registrations.lock().unwrap().active.is_empty()); + // Unexpected exit preserves the desired endpoint for a possible restart. + assert!(provider.inner.own_endpoint.lock().unwrap().is_some()); + assert!(provider.shutdown().await.is_err()); + assert!(provider.inner.own_endpoint.lock().unwrap().is_none()); + assert!(provider.shutdown().await.is_err()); + assert!(provider.watch().await.is_err()); + } + + #[tokio::test] + async fn panic_clears_snapshot_and_delayed_cleanup_blocks_restart_until_one_new_generation() { + assert_mdns_finalization_blocks_restart(true).await; + } + + #[tokio::test] + async fn controlled_browse_error_blocks_restart_until_delayed_cleanup_finishes() { + assert_mdns_finalization_blocks_restart(false).await; + } + + #[tokio::test] + async fn browse_owner_serializes_name_changes_reannouncements_and_shutdown() { + let config = MdnsDiscoveryConfig::new("actor"); + let cleanup = fake_cleanup(); + let daemon = Arc::clone(&cleanup.daemon.state); + let state = Arc::new(DynamicState::new(8)); + let endpoint = Arc::new(StdMutex::new(None)); + let (events, event_rx) = mpsc::channel(8); + let (monitor, monitor_rx) = mpsc::channel(8); + let (announcements, announcement_rx) = mpsc::channel(8); + let (stop, stop_rx, stop_deadline_rx) = shutdown_channels(); + let task = start_mdns_task( + config.clone(), + Arc::clone(&state), + Arc::clone(&endpoint), + FakeEvents { + receiver: event_rx, + processed: None, + }, + FakeEvents { + receiver: monitor_rx, + processed: None, + }, + cleanup, + announcement_rx, + stop_rx, + stop_deadline_rx, + ); + let (reply, response) = oneshot::channel(); + announcements + .send(AnnounceRequest { + endpoint: "127.0.0.1:9000".into(), + reply, + }) + .await + .unwrap(); + response.await.unwrap().unwrap(); + let original = daemon.lock().unwrap().active.keys().next().unwrap().clone(); + let mut alias_config = config.clone(); + alias_config.instance_name = "actor (2)".into(); + let (service, alias) = build_service( + &alias_config, + &cluster_service_type(&config.cluster_id), + "127.0.0.2:9000", + ) + .unwrap(); + let resolved = service.as_resolved_service(); + // Simulate .local auto-address resolution preceding its monitor event. + deliver( + &events, + ServiceEvent::ServiceResolved(Box::new(resolved.clone())), + ) + .await; + assert_eq!(state.snapshot().peers(), ["127.0.0.2:9000"]); + deliver( + &monitor, + DaemonEvent::NameChange(DnsNameChange { + original: original.clone(), + new_name: alias.clone(), + rr_type: RRType::SRV, + intf_name: "controlled".into(), + }), + ) + .await; + assert!(state.snapshot().peers().is_empty()); + let (reply, response) = oneshot::channel(); + announcements + .send(AnnounceRequest { + endpoint: "127.0.0.1:9001".into(), + reply, + }) + .await + .unwrap(); + response.await.unwrap().unwrap(); + assert_eq!(endpoint.lock().unwrap().as_deref(), Some("127.0.0.1:9001")); + let replacement = daemon.lock().unwrap().active.keys().next().unwrap().clone(); + assert_ne!(original, replacement); + assert_eq!(daemon.lock().unwrap().active.len(), 1); + deliver(&events, ServiceEvent::ServiceResolved(Box::new(resolved))).await; + assert!(state.snapshot().peers().is_empty()); + // Another reannouncement (unchanged endpoint) still retires its key. + let (reply, response) = oneshot::channel(); + announcements + .send(AnnounceRequest { + endpoint: "127.0.0.1:9001".into(), + reply, + }) + .await + .unwrap(); + response.await.unwrap().unwrap(); + assert_eq!(daemon.lock().unwrap().active.len(), 1); + assert!(!daemon.lock().unwrap().active.contains_key(&replacement)); + request_shutdown(&stop, SHUTDOWN_BUDGET); + // A request queued concurrently with shutdown must never register. + let (reply, response) = oneshot::channel(); + announcements + .send(AnnounceRequest { + endpoint: "127.0.0.1:9002".into(), + reply, + }) + .await + .unwrap(); + assert!(response.await.unwrap().is_err()); + tokio::time::timeout(SHUTDOWN_BUDGET, task.join()) + .await + .unwrap() + .unwrap(); + assert!(endpoint.lock().unwrap().is_none()); + let daemon = daemon.lock().unwrap(); + assert!(daemon.active.is_empty()); + assert!( + !daemon + .calls + .iter() + .any(|call| call == &format!("unregister:{alias}")) + ); + assert_eq!(daemon.calls.last().unwrap(), "shutdown"); + } + + fn rename_event(owned: &mut OwnedAnnouncements, original: &str, alias: &str) { + let event = DaemonEvent::NameChange(DnsNameChange { + original: original.into(), + new_name: alias.into(), + rr_type: RRType::SRV, + intf_name: "controlled".into(), + }); + if let DaemonEvent::NameChange(change) = event { + assert!(owned.name_change(&change).unwrap()); + } + } + + #[tokio::test] + async fn renamed_reannouncement_withdraws_original_key_and_filters_late_aliases() { + let config = MdnsDiscoveryConfig::new("Node"); + let mut cleanup = fake_cleanup(); + let old = "127.0.0.1:9000"; + let new = "127.0.0.1:9001"; + replace_announcement(&mut cleanup, &config, old.into()) + .await + .unwrap(); + let original = cleanup.owned.current.as_ref().unwrap().key.clone(); + let alias = "Node (2)._numax._tcp.local."; + rename_event(&mut cleanup.owned, &original, alias); + let mut instances = HashMap::from([ + (alias.into(), instance(vec![old.into()])), + ("foreign".into(), instance(vec!["127.0.0.1:9999".into()])), + ]); + let mut order = vec![alias.into(), "foreign".into()]; + replace_announcement(&mut cleanup, &config, new.into()) + .await + .unwrap(); + let replacement = cleanup.owned.current.as_ref().unwrap().key.clone(); + assert_ne!(original, replacement); + { + let daemon = cleanup.daemon.state.lock().unwrap(); + assert_eq!(daemon.active.len(), 1); + assert_eq!(daemon.active[&replacement].get_port(), 9001); + assert_eq!( + daemon.calls, + [ + format!("register:{original}"), + format!("register:{replacement}"), + format!("unregister:{original}") + ] + ); + } + // Delayed per-interface renames must still match the retired original. + rename_event(&mut cleanup.owned, &original, "Node (3)._numax._tcp.local."); + assert!(cleanup.owned.matches(alias, &[])); + assert!(cleanup.owned.matches(&original, &[])); + assert!(cleanup.owned.matches(&replacement, &[])); + assert!(cleanup.owned.matches("unknown", &[old.into()])); + assert!(cleanup.owned.matches("unknown", &[new.into()])); + remove_owned_instances(&cleanup.owned, &mut instances, &mut order); + assert_eq!(order, ["foreign"]); + let state = DynamicState::new(8); + publish_instances(&state, &instances, &order, 8); + assert_eq!(flatten_instances(&instances, &order, 8), ["127.0.0.1:9999"]); + shutdown_daemon(&mut cleanup, SHUTDOWN_BUDGET) + .await + .unwrap(); + let daemon = cleanup.daemon.state.lock().unwrap(); + assert!(daemon.active.is_empty()); + assert_eq!( + &daemon.calls[3..], + [format!("unregister:{replacement}"), "shutdown".into()] + ); + assert!(cleanup.owned.names.is_empty()); + } + + #[tokio::test] + async fn failed_registration_preserves_previous_key_endpoint_and_alias() { + let config = MdnsDiscoveryConfig::new("rollback"); + let mut cleanup = fake_cleanup(); + replace_announcement(&mut cleanup, &config, "127.0.0.1:9000".into()) + .await + .unwrap(); + let original = cleanup.owned.current.as_ref().unwrap().key.clone(); + let alias = "rollback (2)._numax._tcp.local."; + rename_event(&mut cleanup.owned, &original, alias); + cleanup.daemon.state.lock().unwrap().fail_register = true; + assert!( + replace_announcement(&mut cleanup, &config, "127.0.0.1:9001".into()) + .await + .is_err() + ); + assert_eq!(cleanup.owned.current.as_ref().unwrap().key, original); + assert_eq!( + cleanup.owned.current.as_ref().unwrap().endpoint, + "127.0.0.1:9000" + ); + assert!(cleanup.owned.matches(alias, &[])); + assert!(!cleanup.owned.matches("unknown", &["127.0.0.1:9001".into()])); + assert_eq!( + cleanup.daemon.state.lock().unwrap().calls, + [format!("register:{original}")] + ); + shutdown_daemon(&mut cleanup, SHUTDOWN_BUDGET) + .await + .unwrap(); + assert!(cleanup.daemon.state.lock().unwrap().active.is_empty()); + } + + #[tokio::test] + async fn failed_retirement_retains_both_keys_for_acknowledged_cleanup() { + let config = MdnsDiscoveryConfig::new("retirement"); + let mut cleanup = fake_cleanup(); + replace_announcement(&mut cleanup, &config, "127.0.0.1:9000".into()) + .await + .unwrap(); + cleanup.daemon.state.lock().unwrap().fail_withdraw = true; + assert!( + replace_announcement(&mut cleanup, &config, "127.0.0.1:9001".into()) + .await + .is_err() + ); + assert_eq!(cleanup.owned.keys.len(), 2); + assert!( + replace_announcement(&mut cleanup, &config, "127.0.0.1:9002".into()) + .await + .is_err() + ); + let keys = cleanup.owned.keys.clone(); + cleanup.daemon.state.lock().unwrap().fail_withdraw = false; + shutdown_daemon(&mut cleanup, SHUTDOWN_BUDGET) + .await + .unwrap(); + let daemon = cleanup.daemon.state.lock().unwrap(); + assert!(daemon.active.is_empty()); + for key in keys { + assert!(daemon.calls[3..].contains(&format!("unregister:{key}"))); + } + assert_eq!(daemon.calls.last().unwrap(), "shutdown"); + } + + #[tokio::test] + async fn cancelled_announcement_waiter_does_not_cancel_retirement_or_shutdown() { + let config = MdnsDiscoveryConfig::new("cancel"); + let mut cleanup = fake_cleanup(); + replace_announcement(&mut cleanup, &config, "127.0.0.1:9000".into()) + .await + .unwrap(); + let original = cleanup.owned.current.as_ref().unwrap().key.clone(); + rename_event( + &mut cleanup.owned, + &original, + "cancel (2)._numax._tcp.local.", + ); + let (started, entered) = oneshot::channel(); + let (ack, release) = oneshot::channel(); + cleanup.daemon.withdrawal = Some((started, release)); + let state = Arc::clone(&cleanup.daemon.state); + let provider = Arc::new(MdnsDiscovery::new(config.clone()).unwrap()); + let (_events, event_rx) = mpsc::channel(8); + let (_monitor, monitor_rx) = mpsc::channel(8); + let (announcements, announcement_rx) = mpsc::channel(8); + let (stop, stop_rx, stop_deadline_rx) = shutdown_channels(); + { + let mut lifecycle = provider.inner.lifecycle.lock().unwrap(); + let task = start_mdns_task( + config, + Arc::clone(&provider.inner.state), + Arc::clone(&provider.inner.own_endpoint), + FakeEvents { + receiver: event_rx, + processed: None, + }, + FakeEvents { + receiver: monitor_rx, + processed: None, + }, + cleanup, + announcement_rx, + stop_rx, + stop_deadline_rx, + ); + lifecycle.task = Some(task); + lifecycle.announcements = Some(announcements); + lifecycle.shutdown = Some(stop); + } + let caller = Arc::clone(&provider); + let waiter = tokio::spawn(async move { + caller + .announce(&PeerAnnouncement { + endpoint: "127.0.0.1:9001".into(), + }) + .await + }); + assert_eq!(entered.await.unwrap(), original); + assert_eq!(state.lock().unwrap().active.len(), 2); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + provider.request_shutdown(); + assert!( + provider + .announce(&PeerAnnouncement { + endpoint: "127.0.0.1:9002".into() + }) + .await + .is_err() + ); + assert!( + !provider + .inner + .lifecycle + .lock() + .unwrap() + .task + .as_ref() + .unwrap() + .completion_ready() + ); + ack.send(()).unwrap(); + tokio::time::timeout(SHUTDOWN_BUDGET, provider.shutdown()) + .await + .unwrap() + .unwrap(); + let state = state.lock().unwrap(); + assert!(state.active.is_empty()); + assert_eq!(state.calls.last().unwrap(), "shutdown"); + } + + #[tokio::test] + async fn shutdown_during_replacement_uses_one_deadline_and_cleans_both_keys() { + let config = MdnsDiscoveryConfig::new("shared-deadline"); + let mut cleanup = fake_cleanup(); + replace_announcement(&mut cleanup, &config, "127.0.0.1:9000".into()) + .await + .unwrap(); + let original = cleanup.owned.current.as_ref().unwrap().key.clone(); + let (started, entered) = oneshot::channel(); + let (_ack, missing_ack) = oneshot::channel(); + cleanup.daemon.withdrawal = Some((started, missing_ack)); + cleanup.daemon.missing_withdraw_acks = true; + let registrations = Arc::clone(&cleanup.daemon.state); + let provider = Arc::new(MdnsDiscovery::new(config).unwrap()); + let (generation, _events, _monitor) = fake_generation(&provider, cleanup); + let completion = generation.task.clone(); + provider.ensure_started_with(|| Ok(generation)).unwrap(); + + let caller = Arc::clone(&provider); + let announcement = tokio::spawn(async move { + caller + .announce(&PeerAnnouncement { + endpoint: "127.0.0.1:9001".into(), + }) + .await + }); + assert_eq!(entered.await.unwrap(), original); + assert_eq!(registrations.lock().unwrap().active.len(), 2); + + let budget = Duration::from_millis(120); + provider.request_shutdown_with_budget(budget); + let first_deadline = provider + .inner + .lifecycle + .lock() + .unwrap() + .shutdown + .as_ref() + .and_then(|shutdown| *shutdown.deadline.borrow()) + .unwrap(); + assert_eq!( + announcement.await.unwrap(), + Err(provider_error("provider is shut down", false)) + ); + provider.request_shutdown_with_budget(Duration::from_secs(30)); + let repeated_deadline = provider + .inner + .lifecycle + .lock() + .unwrap() + .shutdown + .as_ref() + .and_then(|shutdown| *shutdown.deadline.borrow()) + .unwrap(); + assert_eq!(repeated_deadline, first_deadline); + + let result = tokio::time::timeout(Duration::from_millis(500), provider.shutdown()) + .await + .expect("shutdown renewed its deadline"); + assert!(result.is_err()); + assert!(completion.completion_ready()); + let state = registrations.lock().unwrap(); + let unregisters: Vec<_> = state + .calls + .iter() + .filter(|call| call.starts_with("unregister:")) + .collect(); + assert_eq!(unregisters.len(), 2); + assert_ne!(unregisters[0], unregisters[1]); + assert_eq!(state.calls.last().unwrap(), "shutdown"); + assert!(state.active.is_empty()); + } + + #[tokio::test] + async fn alias_history_is_bounded_and_does_not_discard_owned_names() { + let mut cleanup = fake_cleanup(); + let config = MdnsDiscoveryConfig::new("bounded"); + replace_announcement(&mut cleanup, &config, "127.0.0.1:9000".into()) + .await + .unwrap(); + let original = cleanup.owned.current.as_ref().unwrap().key.clone(); + for index in 1..MAX_OWN_HISTORY { + rename_event( + &mut cleanup.owned, + &original, + &format!("bounded ({index})._numax._tcp.local."), + ); + } + assert_eq!(cleanup.owned.names.len(), MAX_OWN_HISTORY); + assert!( + cleanup + .owned + .name_change(&DnsNameChange { + original: original.clone(), + new_name: "overflow._numax._tcp.local.".into(), + rr_type: RRType::SRV, + intf_name: "controlled".into(), + }) + .is_err() + ); + assert!( + replace_announcement(&mut cleanup, &config, "127.0.0.1:9001".into()) + .await + .is_err() + ); + assert!(cleanup.owned.matches(&original, &[])); + assert_eq!(cleanup.owned.keys.len(), 1); + shutdown_daemon(&mut cleanup, SHUTDOWN_BUDGET) + .await + .unwrap(); + assert!(cleanup.daemon.state.lock().unwrap().active.is_empty()); + } + + #[tokio::test] + #[ignore = "requires local multicast mDNS networking"] + async fn two_daemons_discover_and_remove_an_announced_endpoint() { + let nonce = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let suffix = format!("{}-{nonce}", std::process::id()); + let cluster = format!("mdns-test-{suffix}"); + let mut publisher_config = MdnsDiscoveryConfig::new(format!("publisher-{suffix}")); + publisher_config.cluster_id = cluster.clone(); + let mut observer_config = MdnsDiscoveryConfig::new(format!("observer-{suffix}")); + observer_config.cluster_id = cluster; + let publisher = MdnsDiscovery::new(publisher_config).unwrap(); + let observer = MdnsDiscovery::new(observer_config).unwrap(); + let endpoint = "127.0.0.1:43111"; + let mut watch = observer.watch().await.unwrap(); + + publisher + .announce(&PeerAnnouncement { + endpoint: endpoint.into(), + }) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if super::super::observed_peers(watch.recv().await.unwrap().change) + == vec![endpoint.to_string()] + { + break; + } + } + }) + .await + .unwrap(); + + let replacement = "127.0.0.1:43112"; + publisher + .announce(&PeerAnnouncement { + endpoint: replacement.into(), + }) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if super::super::observed_peers(watch.recv().await.unwrap().change) + == vec![replacement.to_string()] + { + break; + } + } + }) + .await + .unwrap(); + assert!(publisher.discover().await.unwrap().peers().is_empty()); + + publisher.shutdown().await.unwrap(); + tokio::time::timeout(Duration::from_secs(10), async { + loop { + if super::super::observed_peers(watch.recv().await.unwrap().change).is_empty() { + break; + } + } + }) + .await + .unwrap(); + + observer.shutdown().await.unwrap(); + } +} diff --git a/crates/nx-core/src/lib.rs b/crates/nx-core/src/lib.rs index a0879bf..5a4d895 100644 --- a/crates/nx-core/src/lib.rs +++ b/crates/nx-core/src/lib.rs @@ -1,4 +1,5 @@ pub mod control; +pub mod discovery; pub mod host_api; pub mod observability; pub mod runtime; @@ -9,6 +10,19 @@ pub use control::{ ControlError, ControlPage, ModuleInfo, ModuleRegistration, PeerInfo, RuntimeControl, RuntimeControlHandle, RuntimeIntrospection, RuntimeManagement, SharedRuntimeControl, }; -pub use nx_net::{SerializationFormat, TlsConfig}; +pub use discovery::{ + AnnouncementSupport, BootstrapDiscoverySettings, BootstrapGossipDiscovery, + BootstrapGossipDiscoveryConfig, DEFAULT_DISCOVERY_CLUSTER, DEFAULT_DISCOVERY_EVENT_CAPACITY, + DEFAULT_MAX_PEER_CANDIDATES, DiscoveryChange, DiscoveryError, DiscoveryEvent, + DiscoveryProvider, DiscoveryRuntimeConfig, DiscoverySnapshot, DiscoveryWatch, DnsSrvDiscovery, + DnsSrvDiscoveryConfig, DnsSrvDiscoverySettings, FileDiscoverySettings, FileWatchDiscovery, + FileWatchDiscoveryConfig, MAX_DISCOVERY_EVENT_CAPACITY, MdnsDiscovery, MdnsDiscoveryConfig, + MdnsDiscoverySettings, PeerAnnouncement, PeerDiscovery, RuntimeDiscoveryConfig, + RuntimeDiscoveryMode, StaticDiscovery, +}; +pub use nx_net::{ + BootstrapClientConfig, ConnectionDirection, MAX_BOOTSTRAP_RESPONSE_CAPACITY, + PeerConnectionInfo, PeerIdentity, PeerIdentityVerification, SerializationFormat, TlsConfig, +}; pub use observability::ObservabilityConfig; -pub use sync_config::SyncConfig; +pub use sync_config::{SyncConfig, SyncConfigError}; diff --git a/crates/nx-core/src/runtime.rs b/crates/nx-core/src/runtime.rs index 4a9e045..3036d96 100644 --- a/crates/nx-core/src/runtime.rs +++ b/crates/nx-core/src/runtime.rs @@ -138,6 +138,18 @@ pub struct Runtime { impl Runtime { pub fn new(config: RuntimeConfig) -> Result { + let mut discovery = crate::RuntimeDiscoveryConfig::default(); + if let Some(sync) = &config.sync { + discovery.max_candidates = discovery.max_candidates.max(sync.peers.len()); + } + Self::new_with_discovery(config, discovery) + } + + /// Create a runtime with an explicitly resolved peer discovery policy. + pub fn new_with_discovery( + config: RuntimeConfig, + discovery: crate::RuntimeDiscoveryConfig, + ) -> Result { // Engine: async support is required so wasmtime can yield across host calls let mut wasm_cfg = wasmtime::Config::new(); wasm_cfg.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable); @@ -187,11 +199,15 @@ impl Runtime { // Initialize SyncManager if configured, and derive its handle up-front so every HostState built afterwards sees the same op channel. let (sync_manager, sync_handle) = if let Some(ref sync_config) = config.sync { let node_id = load_or_create_node_id(&store)?; - let manager = SyncManager::try_new( + let (providers, discovery_runtime) = + crate::discovery::build_runtime_discovery(&node_id, sync_config, &discovery)?; + let manager = SyncManager::try_new_with_discovery( node_id, sync_config.clone(), Arc::clone(&store), Arc::clone(&metrics), + providers, + discovery_runtime, )?; let handle = manager.handle(); (Some(manager), Some(handle)) diff --git a/crates/nx-core/src/sync_config.rs b/crates/nx-core/src/sync_config.rs index a5e9aa2..c6df286 100644 --- a/crates/nx-core/src/sync_config.rs +++ b/crates/nx-core/src/sync_config.rs @@ -1,6 +1,22 @@ use nx_net::{SerializationFormat, TlsConfig}; use std::time::Duration; +#[derive(Debug)] +#[non_exhaustive] +pub enum SyncConfigError { + Invalid(String), +} + +impl std::fmt::Display for SyncConfigError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Invalid(message) => formatter.write_str(message), + } + } +} + +impl std::error::Error for SyncConfigError {} + /// Default maximum number of simultaneously connected peers. pub const DEFAULT_MAX_PEERS: usize = nx_net::DEFAULT_MAX_PEERS; @@ -99,6 +115,43 @@ impl Default for SyncConfig { } impl SyncConfig { + /// Validate allocation bounds and timer deadlines before starting services. + /// Zero retry delays and anti-entropy intervals retain their 1 ms normalization. + pub fn validate(&self) -> Result<(), SyncConfigError> { + for (name, limit) in [ + ("queued_ops_limit", self.queued_ops_limit), + ("max_peers", self.max_peers), + ] { + if limit > tokio::sync::Semaphore::MAX_PERMITS { + return Err(SyncConfigError::Invalid(format!( + "{name} exceeds the supported channel/semaphore capacity" + ))); + } + } + let now = std::time::Instant::now(); + for (name, duration) in [ + ("reconnect_initial_delay", self.reconnect_initial_delay), + ("reconnect_max_delay", self.reconnect_max_delay), + ("anti_entropy_interval", self.anti_entropy_interval), + ("socket_timeout", self.socket_timeout), + ] { + if now + .checked_add(duration.max(Duration::from_millis(1))) + .is_none() + { + return Err(SyncConfigError::Invalid(format!( + "{name} exceeds the supported deadline range" + ))); + } + } + if self.socket_timeout.is_zero() { + return Err(SyncConfigError::Invalid( + "socket_timeout must be positive".into(), + )); + } + Ok(()) + } + pub fn new() -> Self { Self::default() } @@ -179,6 +232,18 @@ impl SyncConfig { mod tests { use super::*; + #[test] + fn validation_preserves_zero_normalization_and_disabled_peer_admission() { + SyncConfig::default().validate().unwrap(); + SyncConfig::new() + .with_max_peers(0) + .with_queued_ops_limit(0) + .with_reconnect_backoff(Duration::ZERO, Duration::ZERO) + .with_anti_entropy_interval(Duration::ZERO) + .validate() + .unwrap(); + } + #[test] fn test_is_enabled_requires_listen() { let cfg = SyncConfig::new(); diff --git a/crates/nx-core/src/sync_manager/candidates.rs b/crates/nx-core/src/sync_manager/candidates.rs new file mode 100644 index 0000000..2a34101 --- /dev/null +++ b/crates/nx-core/src/sync_manager/candidates.rs @@ -0,0 +1,1967 @@ +use std::collections::{HashMap, HashSet}; +use std::future::pending; +use std::net::{IpAddr, SocketAddr}; +use std::sync::Arc; +use std::time::{Duration, Instant as StdInstant}; + +use tokio::sync::{mpsc, oneshot, watch}; +use tokio::task::JoinHandle; +use tokio::time::Instant as TokioInstant; +use tracing::{debug, warn}; + +use crate::discovery::{ + AbortOnDropTask, AnnouncementSupport, DiscoveryChange, DiscoveryError, DiscoveryProvider, + DiscoveryRuntimeConfig, DiscoverySnapshot, DiscoveryWatch, PeerAnnouncement, +}; + +const DISCOVERY_RETRY_INITIAL_DELAY: Duration = Duration::from_millis(500); +const DISCOVERY_RETRY_MAX_DELAY: Duration = Duration::from_secs(30); +const DISCOVERY_OPERATION_TIMEOUT: Duration = Duration::from_secs(5); + +#[derive(Debug, Clone, Copy)] +#[cfg_attr(test, derive(PartialEq, Eq))] +struct CandidateContribution { + expires_at: Option, +} + +#[derive(Debug, Clone, Default)] +#[cfg_attr(test, derive(PartialEq, Eq))] +struct CandidateRecord { + sources: HashMap, +} + +#[derive(Debug, Clone)] +#[cfg_attr(test, derive(PartialEq, Eq))] +struct CandidateRegistry { + max_candidates: usize, + order: Vec, + source_order: Vec, + source_candidates: HashMap>, + records: HashMap, + local_endpoints: HashSet, + #[cfg(test)] + order_rebuilds: usize, +} + +impl CandidateRegistry { + fn new(max_candidates: usize) -> Result { + if max_candidates == 0 { + return Err(configuration_error( + "coordinator", + "max_candidates must be greater than zero", + )); + } + Ok(Self { + max_candidates, + order: Vec::new(), + source_order: Vec::new(), + source_candidates: HashMap::new(), + records: HashMap::new(), + local_endpoints: HashSet::new(), + #[cfg(test)] + order_rebuilds: 0, + }) + } + + fn endpoints(&self) -> Arc> { + Arc::new( + self.order + .iter() + .filter(|endpoint| self.records.contains_key(*endpoint)) + .cloned() + .collect(), + ) + } + + fn replace_source( + &mut self, + source_id: &str, + peers: &[String], + ttl: Option, + now: StdInstant, + ) -> Result { + if peers.len() > self.max_candidates { + return Err(configuration_error( + "coordinator", + format!( + "discovery snapshot exceeds the {} candidate limit", + self.max_candidates + ), + )); + } + let mut canonical = Vec::new(); + let mut seen = HashSet::new(); + for peer in peers { + let endpoint = match canonicalize_endpoint(peer) { + Ok(endpoint) => endpoint, + Err(error) => { + warn!( + source = %source_id, + endpoint = %peer, + error = %error, + "rejected invalid discovery snapshot candidate" + ); + continue; + } + }; + if seen.insert(endpoint.clone()) { + canonical.push(endpoint); + } + } + // Empty, invalid-only and local-only snapshots do not acquire a lease. + let expires_at = match ttl { + Some(ttl) + if canonical + .iter() + .any(|peer| !self.local_endpoints.contains(peer)) => + { + Some(now.checked_add(ttl).ok_or_else(|| { + configuration_error(source_id, "candidate_ttl exceeds the platform time range") + })?) + } + _ => None, + }; + self.replace_contributions( + source_id, + canonical + .into_iter() + .map(|endpoint| (endpoint, CandidateContribution { expires_at })) + .collect(), + ) + } + + fn replace_snapshot( + &mut self, + source_id: &str, + snapshot: &DiscoverySnapshot, + ttl: Option, + now: StdInstant, + ) -> Result { + let Some(observations) = snapshot.observations() else { + return self.replace_source(source_id, snapshot.peers(), ttl, now); + }; + if snapshot.peers().len() > self.max_candidates { + return Err(configuration_error( + source_id, + "discovery snapshot exceeds candidate limit", + )); + } + let mut positions = HashMap::::new(); + let mut contributions: Vec<(String, CandidateContribution)> = Vec::new(); + for (peer, at) in snapshot.peers().iter().zip(observations) { + let expires_at = if let Some(ttl) = ttl { + let deadline = at.checked_add(ttl).ok_or_else(|| { + configuration_error(source_id, "candidate_ttl exceeds the platform time range") + })?; + if deadline <= now { + continue; + } + Some(deadline) + } else { + None + }; + if let Ok(peer) = canonicalize_endpoint(peer) { + if let Some(&position) = positions.get(&peer) { + // Keep the first live occurrence's position and the newest lease. + let contribution = &mut contributions[position].1; + contribution.expires_at = contribution.expires_at.max(expires_at); + } else { + positions.insert(peer.clone(), contributions.len()); + contributions.push((peer, CandidateContribution { expires_at })); + } + } + } + self.replace_contributions(source_id, contributions) + } + + fn replace_contributions( + &mut self, + source_id: &str, + contributions: Vec<(String, CandidateContribution)>, + ) -> Result { + // Prepare the entire replacement off-registry: a global capacity error + // must not publish removals, reordered sources or partially renewed leases. + let mut updated = self.clone(); + updated.register_source(source_id); + let retained = contributions + .iter() + .map(|(endpoint, _)| endpoint.clone()) + .collect::>(); + updated.remove_source_except(source_id, &retained); + let mut candidates = Vec::with_capacity(contributions.len()); + for (endpoint, contribution) in contributions { + candidates.push(endpoint.clone()); + if updated.local_endpoints.contains(&endpoint) { + continue; + } + if !updated.records.contains_key(&endpoint) + && updated.records.len() >= updated.max_candidates + { + return Err(configuration_error( + "coordinator", + format!("peer candidate limit reached: {}", updated.max_candidates), + )); + } + updated + .records + .entry(endpoint) + .or_default() + .sources + .insert(source_id.to_string(), contribution); + } + updated + .source_candidates + .insert(source_id.to_string(), candidates); + updated.rebuild_order(); + let changed = updated.order != self.order; + *self = updated; + Ok(changed) + } + + fn add( + &mut self, + source_id: &str, + endpoint: String, + ttl: Option, + now: StdInstant, + ) -> Result { + if self.local_endpoints.contains(&endpoint) { + return Ok(false); + } + let is_new = !self.records.contains_key(&endpoint); + if is_new && self.records.len() >= self.max_candidates { + return Err(configuration_error( + "coordinator", + format!("peer candidate limit reached: {}", self.max_candidates), + )); + } + let expires_at = match ttl { + Some(ttl) => Some(now.checked_add(ttl).ok_or_else(|| { + configuration_error(source_id, "candidate_ttl exceeds the platform time range") + })?), + None => None, + }; + let previous_order = self.order.clone(); + self.register_source(source_id); + let source_candidates = self + .source_candidates + .entry(source_id.to_string()) + .or_default(); + if !source_candidates.contains(&endpoint) { + source_candidates.push(endpoint.clone()); + } + self.records + .entry(endpoint.clone()) + .or_default() + .sources + .insert(source_id.to_string(), CandidateContribution { expires_at }); + self.rebuild_order(); + Ok(self.order != previous_order) + } + + fn remove(&mut self, source_id: &str, endpoint: &str) -> bool { + let before = self.endpoints(); + if let Some(candidates) = self.source_candidates.get_mut(source_id) { + candidates.retain(|candidate| candidate != endpoint); + } + if let Some(record) = self.records.get_mut(endpoint) { + record.sources.remove(source_id); + } + self.prune_empty(); + self.prune_source_candidates(); + self.rebuild_order(); + self.endpoints() != before + } + + fn remove_source_except(&mut self, source_id: &str, retained: &HashSet) { + for (endpoint, record) in &mut self.records { + if !retained.contains(endpoint) { + record.sources.remove(source_id); + } + } + self.prune_empty(); + } + + fn source_unavailable(&mut self, source_id: &str, leased: bool) -> bool { + if leased { + return false; + } + let before = self.endpoints(); + self.source_candidates.remove(source_id); + for record in self.records.values_mut() { + record.sources.remove(source_id); + } + self.prune_empty(); + self.prune_source_candidates(); + self.rebuild_order(); + self.endpoints() != before + } + + fn set_local_endpoints(&mut self, endpoints: Vec) -> bool { + let before = self.endpoints(); + self.local_endpoints.clear(); + for endpoint in endpoints { + self.local_endpoints.insert(endpoint); + } + self.records + .retain(|endpoint, _| !self.local_endpoints.contains(endpoint)); + self.prune_source_candidates(); + self.rebuild_order(); + self.endpoints() != before + } + + fn expire(&mut self, now: StdInstant) -> bool { + let before = self.endpoints(); + for record in self.records.values_mut() { + record + .sources + .retain(|_, source| source.expires_at.is_none_or(|deadline| deadline > now)); + } + self.prune_empty(); + self.prune_source_candidates(); + self.rebuild_order(); + self.endpoints() != before + } + + fn next_expiry(&self) -> Option { + self.records + .values() + .flat_map(|record| record.sources.values()) + .filter_map(|source| source.expires_at) + .min() + } + + fn prune_empty(&mut self) -> bool { + let before = self.records.len(); + self.records.retain(|_, record| !record.sources.is_empty()); + self.records.len() != before + } + + fn register_source(&mut self, source_id: &str) { + if !self.source_order.iter().any(|known| known == source_id) { + self.source_order.push(source_id.to_string()); + } + } + + fn prune_source_candidates(&mut self) { + for (source_id, candidates) in &mut self.source_candidates { + candidates.retain(|endpoint| { + self.records + .get(endpoint) + .is_some_and(|record| record.sources.contains_key(source_id)) + }); + } + self.source_candidates + .retain(|_, candidates| !candidates.is_empty()); + } + + fn rebuild_order(&mut self) { + #[cfg(test)] + { + self.order_rebuilds += 1; + } + let mut order = Vec::with_capacity(self.records.len()); + let mut seen = HashSet::with_capacity(self.records.len()); + for source_id in &self.source_order { + let Some(candidates) = self.source_candidates.get(source_id) else { + continue; + }; + for endpoint in candidates { + if self + .records + .get(endpoint) + .is_some_and(|record| record.sources.contains_key(source_id)) + && seen.insert(endpoint) + { + order.push(endpoint.clone()); + } + } + } + self.order = order; + } +} + +enum CandidateCommand { + Snapshot { + source_id: String, + snapshot: DiscoverySnapshot, + ttl: Option, + }, + ReplaceSource { + source_id: String, + peers: Vec, + ttl: Option, + }, + Add { + source_id: String, + endpoint: String, + ttl: Option, + }, + Remove { + source_id: String, + endpoint: String, + }, + SourceUnavailable { + source_id: String, + leased: bool, + }, + SetLocalEndpoints { + endpoints: Vec, + reply: oneshot::Sender<()>, + }, +} + +pub(super) struct DiscoveryCoordinator { + config: DiscoveryRuntimeConfig, + providers: Vec, + candidates_rx: watch::Receiver>>, + command_tx: mpsc::Sender, + shutdown_tx: watch::Sender, + coordinator_task: Option>, + provider_tasks: Vec>, +} + +impl Drop for DiscoveryCoordinator { + fn drop(&mut self) { + self.request_shutdown(); + for task in &self.provider_tasks { + task.abort(); + } + if let Some(task) = &self.coordinator_task { + task.abort(); + } + } +} + +impl DiscoveryCoordinator { + pub(super) async fn start( + providers: Vec, + config: DiscoveryRuntimeConfig, + ) -> Result { + validate_discovery_config(&providers, &config)?; + + let mut registry = CandidateRegistry::new(config.max_candidates())?; + let mut initial_watches = Vec::with_capacity(providers.len()); + for source in &providers { + let provider_watch = + match tokio::time::timeout(DISCOVERY_OPERATION_TIMEOUT, source.provider().watch()) + .await + { + Ok(Ok(provider_watch)) => provider_watch, + Ok(Err(error)) => { + rollback_providers(&providers).await; + return Err(error); + } + Err(_) => { + rollback_providers(&providers).await; + return Err(provider_timeout(source.source_id(), "watch")); + } + }; + if let Err(error) = registry.replace_snapshot( + source.source_id(), + provider_watch.snapshot(), + source.candidate_ttl(), + StdInstant::now(), + ) { + rollback_providers(&providers).await; + return Err(error); + } + initial_watches.push(provider_watch); + } + + let (candidates_tx, candidates_rx) = watch::channel(registry.endpoints()); + let command_capacity = config.max_candidates().clamp(1, 4096); + let (command_tx, command_rx) = mpsc::channel(command_capacity); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let coordinator_task = Some(tokio::spawn(run_candidate_registry( + registry, + candidates_tx, + command_rx, + shutdown_rx, + ))); + + let provider_tasks = providers + .iter() + .cloned() + .zip(initial_watches) + .map(|(source, provider_watch)| { + tokio::spawn(run_provider_watch( + source, + provider_watch, + command_tx.clone(), + shutdown_tx.subscribe(), + )) + }) + .collect(); + + Ok(Self { + config, + providers, + candidates_rx, + command_tx, + shutdown_tx, + coordinator_task, + provider_tasks, + }) + } + + pub(super) fn candidates(&self) -> watch::Receiver>> { + self.candidates_rx.clone() + } + + pub(super) fn request_shutdown(&self) { + let _ = self.shutdown_tx.send(true); + for source in &self.providers { + source.provider().request_shutdown(); + } + } + + pub(super) async fn configure_local_endpoint( + &self, + bound_addr: SocketAddr, + ) -> Result, DiscoveryError> { + let advertised = + resolve_advertised_endpoint(bound_addr, self.config.advertised_endpoint())?; + let mut local_endpoints = Vec::with_capacity(2); + if !bound_addr.ip().is_unspecified() { + local_endpoints.push(bound_addr.to_string()); + } + if let Some(endpoint) = &advertised + && !local_endpoints.contains(endpoint) + { + local_endpoints.push(endpoint.clone()); + } + let (reply, response) = oneshot::channel(); + self.command_tx + .send(CandidateCommand::SetLocalEndpoints { + endpoints: local_endpoints, + reply, + }) + .await + .map_err(|_| DiscoveryError::WatchClosed)?; + response.await.map_err(|_| DiscoveryError::WatchClosed)?; + Ok(advertised) + } + + pub(super) async fn announce( + &self, + advertised_endpoint: Option<&str>, + ) -> Result<(), DiscoveryError> { + for source in &self.providers { + match source.provider().announcement_support() { + AnnouncementSupport::Unsupported => continue, + AnnouncementSupport::Optional if advertised_endpoint.is_none() => continue, + AnnouncementSupport::Required if advertised_endpoint.is_none() => { + return Err(configuration_error( + source.source_id(), + "a wildcard listener requires an explicit advertised endpoint", + )); + } + AnnouncementSupport::Optional | AnnouncementSupport::Required => {} + } + let Some(endpoint) = advertised_endpoint else { + continue; + }; + tokio::time::timeout( + DISCOVERY_OPERATION_TIMEOUT, + source.provider().announce(&PeerAnnouncement { + endpoint: endpoint.to_string(), + }), + ) + .await + .map_err(|_| provider_timeout(source.source_id(), "announcement"))??; + } + Ok(()) + } + + pub(super) async fn shutdown(&mut self) -> Result<(), DiscoveryError> { + self.request_shutdown(); + for task in self.provider_tasks.drain(..) { + let _ = AbortOnDropTask::new(task).join().await; + } + if let Some(task) = self.coordinator_task.take() { + let _ = AbortOnDropTask::new(task).join().await; + } + + shutdown_providers(&self.providers).await + } +} + +async fn run_candidate_registry( + mut registry: CandidateRegistry, + candidates_tx: watch::Sender>>, + mut command_rx: mpsc::Receiver, + mut shutdown_rx: watch::Receiver, +) { + loop { + let next_expiry = registry.next_expiry(); + tokio::select! { + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + break; + } + } + command = command_rx.recv() => { + let Some(command) = command else { + break; + }; + let command = match command { + CandidateCommand::SetLocalEndpoints { endpoints, reply } => { + let changed = registry.set_local_endpoints(endpoints); + if changed { + candidates_tx.send_replace(registry.endpoints()); + } + let _ = reply.send(()); + continue; + } + command => command, + }; + let changed = apply_candidate_command(&mut registry, command); + if changed { + candidates_tx.send_replace(registry.endpoints()); + } + } + _ = wait_for_expiry(next_expiry) => { + if registry.expire(StdInstant::now()) { + candidates_tx.send_replace(registry.endpoints()); + } + } + } + } + debug!("peer candidate coordinator terminated"); +} + +fn apply_candidate_command(registry: &mut CandidateRegistry, command: CandidateCommand) -> bool { + let now = StdInstant::now(); + match command { + CandidateCommand::Snapshot { + source_id, + snapshot, + ttl, + } => match registry.replace_snapshot(&source_id, &snapshot, ttl, now) { + Ok(changed) => changed, + Err(error) => { + warn!(source = %source_id, %error, "rejected observed discovery snapshot"); + false + } + }, + CandidateCommand::ReplaceSource { + source_id, + peers, + ttl, + } => match registry.replace_source(&source_id, &peers, ttl, now) { + Ok(changed) => changed, + Err(error) => { + warn!(source = %source_id, error = %error, "rejected discovery snapshot"); + false + } + }, + CandidateCommand::Add { + source_id, + endpoint, + ttl, + } => match canonicalize_endpoint(&endpoint) + .and_then(|endpoint| registry.add(&source_id, endpoint, ttl, now)) + { + Ok(changed) => changed, + Err(error) => { + warn!(source = %source_id, error = %error, "rejected discovery candidate"); + false + } + }, + CandidateCommand::Remove { + source_id, + endpoint, + } => match canonicalize_endpoint(&endpoint) { + Ok(endpoint) => registry.remove(&source_id, &endpoint), + Err(error) => { + warn!(source = %source_id, error = %error, "rejected discovery candidate removal"); + false + } + }, + CandidateCommand::SourceUnavailable { source_id, leased } => { + registry.source_unavailable(&source_id, leased) + } + CandidateCommand::SetLocalEndpoints { .. } => false, + } +} + +async fn run_provider_watch( + source: DiscoveryProvider, + mut provider_watch: DiscoveryWatch, + command_tx: mpsc::Sender, + mut shutdown_rx: watch::Receiver, +) { + let mut retry_delay = DISCOVERY_RETRY_INITIAL_DELAY; + loop { + let result = tokio::select! { + _ = shutdown_rx.changed() => break, + result = provider_watch.recv() => result, + }; + + match result { + Ok(event) => { + retry_delay = DISCOVERY_RETRY_INITIAL_DELAY; + let command = match event.change { + DiscoveryChange::Observed(snapshot) => CandidateCommand::Snapshot { + source_id: source.source_id().to_string(), + snapshot, + ttl: source.candidate_ttl(), + }, + DiscoveryChange::Added(endpoint) => CandidateCommand::Add { + source_id: source.source_id().to_string(), + endpoint, + ttl: source.candidate_ttl(), + }, + DiscoveryChange::Removed(endpoint) => CandidateCommand::Remove { + source_id: source.source_id().to_string(), + endpoint, + }, + DiscoveryChange::Replaced(peers) => CandidateCommand::ReplaceSource { + source_id: source.source_id().to_string(), + peers, + ttl: source.candidate_ttl(), + }, + }; + if !send_command(&command_tx, command, &mut shutdown_rx).await { + break; + } + } + Err(error) => { + let leased = source.candidate_ttl().is_some(); + if !send_command( + &command_tx, + CandidateCommand::SourceUnavailable { + source_id: source.source_id().to_string(), + leased, + }, + &mut shutdown_rx, + ) + .await + { + break; + } + if !discovery_error_is_retryable(&error) { + warn!(source = %source.source_id(), error = %error, "discovery watch stopped"); + break; + } + debug!(source = %source.source_id(), error = %error, "resubscribing discovery watch"); + if !wait_for_retry(retry_delay, &mut shutdown_rx).await { + break; + } + retry_delay = retry_delay.saturating_mul(2).min(DISCOVERY_RETRY_MAX_DELAY); + let watch_result = tokio::select! { + _ = shutdown_rx.changed() => break, + result = tokio::time::timeout( + DISCOVERY_OPERATION_TIMEOUT, + source.provider().watch(), + ) => result, + }; + match watch_result { + Ok(Ok(new_watch)) => { + let snapshot = new_watch.snapshot().clone(); + if !send_command( + &command_tx, + CandidateCommand::Snapshot { + source_id: source.source_id().to_string(), + snapshot, + ttl: source.candidate_ttl(), + }, + &mut shutdown_rx, + ) + .await + { + break; + } + provider_watch = new_watch; + retry_delay = DISCOVERY_RETRY_INITIAL_DELAY; + } + Ok(Err(error)) if !discovery_error_is_retryable(&error) => { + warn!(source = %source.source_id(), error = %error, "discovery provider failed permanently"); + break; + } + Ok(Err(error)) => { + debug!(source = %source.source_id(), error = %error, "discovery resubscribe failed"); + } + Err(_) => { + debug!(source = %source.source_id(), "discovery resubscribe timed out"); + } + } + } + } + } + debug!(source = %source.source_id(), "discovery watch task terminated"); +} + +async fn send_command( + command_tx: &mpsc::Sender, + command: CandidateCommand, + shutdown_rx: &mut watch::Receiver, +) -> bool { + tokio::select! { + _ = shutdown_rx.changed() => false, + result = command_tx.send(command) => result.is_ok(), + } +} + +async fn wait_for_retry(delay: Duration, shutdown_rx: &mut watch::Receiver) -> bool { + tokio::select! { + _ = shutdown_rx.changed() => false, + _ = tokio::time::sleep(delay) => true, + } +} + +async fn wait_for_expiry(deadline: Option) { + match deadline { + Some(deadline) => tokio::time::sleep_until(TokioInstant::from_std(deadline)).await, + None => pending::<()>().await, + } +} + +fn discovery_error_is_retryable(error: &DiscoveryError) -> bool { + match error { + DiscoveryError::Provider { retryable, .. } => *retryable, + DiscoveryError::WatchOverflow { .. } + | DiscoveryError::WatchRevision { .. } + | DiscoveryError::WatchInvalidated + | DiscoveryError::WatchClosed => true, + DiscoveryError::InvalidConfiguration { .. } | DiscoveryError::Unsupported { .. } => false, + } +} + +fn validate_discovery_config( + providers: &[DiscoveryProvider], + config: &DiscoveryRuntimeConfig, +) -> Result<(), DiscoveryError> { + validate_identifier("cluster_id", config.cluster_id())?; + if let Some(endpoint) = config.advertised_endpoint() { + let (host, port) = parse_host_port(endpoint, true)?; + canonicalize_host_port(&host, port.max(1))?; + } + let mut source_ids = HashSet::new(); + for source in providers { + validate_identifier("source_id", source.source_id())?; + if !source_ids.insert(source.source_id()) { + return Err(configuration_error( + "coordinator", + format!("duplicate discovery source: {}", source.source_id()), + )); + } + if source.provider().cluster_id() != config.cluster_id() { + return Err(configuration_error( + source.source_id(), + format!( + "provider cluster '{}' does not match local cluster '{}'", + source.provider().cluster_id(), + config.cluster_id() + ), + )); + } + if source.candidate_ttl() == Some(Duration::ZERO) { + return Err(configuration_error( + source.source_id(), + "candidate_ttl must be greater than zero", + )); + } + } + Ok(()) +} + +fn validate_identifier(name: &str, value: &str) -> Result<(), DiscoveryError> { + let valid = !value.is_empty() + && value.len() <= 128 + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_' | b'.')); + if valid { + Ok(()) + } else { + Err(configuration_error( + "coordinator", + format!("{name} must be 1..=128 ASCII letters, digits, '.', '_' or '-'"), + )) + } +} + +fn resolve_advertised_endpoint( + bound_addr: SocketAddr, + configured: Option<&str>, +) -> Result, DiscoveryError> { + match configured { + Some(configured) => { + let (host, port) = parse_host_port(configured, true)?; + let port = if port == 0 { bound_addr.port() } else { port }; + canonicalize_host_port(&host, port).map(Some) + } + None if bound_addr.ip().is_unspecified() => Ok(None), + None => Ok(Some(bound_addr.to_string())), + } +} + +pub(crate) fn canonicalize_endpoint(endpoint: &str) -> Result { + let (host, port) = parse_host_port(endpoint, false)?; + canonicalize_host_port(&host, port) +} + +fn parse_host_port(endpoint: &str, allow_zero_port: bool) -> Result<(String, u16), DiscoveryError> { + if endpoint.trim() != endpoint || endpoint.is_empty() { + return Err(configuration_error( + "coordinator", + format!("invalid peer endpoint: {endpoint:?}"), + )); + } + if let Ok(socket) = endpoint.parse::() { + if !allow_zero_port && socket.port() == 0 { + return Err(configuration_error( + "coordinator", + "peer endpoint port must be greater than zero", + )); + } + return Ok((socket.ip().to_string(), socket.port())); + } + let Some((host, port)) = endpoint.rsplit_once(':') else { + return Err(configuration_error( + "coordinator", + format!("peer endpoint must include a port: {endpoint}"), + )); + }; + let host = host.strip_suffix('.').unwrap_or(host); + if !valid_dns_name(host) { + return Err(configuration_error( + "coordinator", + format!("invalid peer endpoint host: {host:?}"), + )); + } + let port = port.parse::().map_err(|_| { + configuration_error( + "coordinator", + format!("invalid peer endpoint port: {port:?}"), + ) + })?; + if !allow_zero_port && port == 0 { + return Err(configuration_error( + "coordinator", + "peer endpoint port must be greater than zero", + )); + } + Ok((host.to_ascii_lowercase(), port)) +} + +fn valid_dns_name(host: &str) -> bool { + !host.is_empty() + && host.len() <= 253 + && !host.contains(':') + && host.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && label + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + && label + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && label + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric) + }) +} + +fn canonicalize_host_port(host: &str, port: u16) -> Result { + if port == 0 { + return Err(configuration_error( + "coordinator", + "advertised endpoint resolved to port zero", + )); + } + if let Ok(ip) = host.parse::() { + let undialable = ip.is_unspecified() + || ip.is_multicast() + || matches!(ip, IpAddr::V4(address) if address.is_broadcast()) + || matches!(ip, IpAddr::V6(address) if address.is_unicast_link_local()); + if undialable { + return Err(configuration_error( + "coordinator", + "peer endpoint must use a dialable unicast IP address", + )); + } + return Ok(SocketAddr::new(ip, port).to_string()); + } + Ok(format!("{}:{port}", host.to_ascii_lowercase())) +} + +fn configuration_error(provider: &str, message: impl Into) -> DiscoveryError { + DiscoveryError::InvalidConfiguration { + provider: provider.to_string(), + message: message.into(), + } +} + +fn provider_timeout(provider: &str, operation: &str) -> DiscoveryError { + DiscoveryError::Provider { + provider: provider.to_string(), + message: format!("{operation} timed out"), + retryable: true, + } +} + +async fn shutdown_providers(providers: &[DiscoveryProvider]) -> Result<(), DiscoveryError> { + let mut tasks = tokio::task::JoinSet::new(); + for source in providers { + source.provider().request_shutdown(); + let source_id = source.source_id().to_string(); + let provider = Arc::clone(source.provider()); + tasks.spawn(async move { + tokio::time::timeout(DISCOVERY_OPERATION_TIMEOUT, provider.shutdown()) + .await + .map_err(|_| provider_timeout(&source_id, "shutdown"))? + }); + } + + let mut first_error = None; + while let Some(result) = tasks.join_next().await { + let result = match result { + Ok(result) => result, + Err(error) => Err(DiscoveryError::Provider { + provider: "coordinator".to_string(), + message: format!("shutdown task failed: {error}"), + retryable: false, + }), + }; + if let Err(error) = result + && first_error.is_none() + { + first_error = Some(error); + } + } + first_error.map_or(Ok(()), Err) +} + +async fn rollback_providers(providers: &[DiscoveryProvider]) { + if let Err(error) = shutdown_providers(providers).await { + warn!(error = %error, "discovery provider rollback failed"); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex as StdMutex; + + #[test] + fn bulk_replacement_preserves_canonical_first_occurrence_and_source_priority() { + let mut registry = CandidateRegistry::new(8).unwrap(); + let now = StdInstant::now(); + registry.set_local_endpoints(vec!["local:1".into()]); + registry + .replace_source("first", &["old:1".into(), "shared:1".into()], None, now) + .unwrap(); + registry + .replace_source("second", &["other:1".into(), "shared:1".into()], None, now) + .unwrap(); + + assert!( + registry + .replace_source( + "first", + &[ + "B.:1".into(), + "Shared:1".into(), + "b:1".into(), + "A:1".into(), + "shared.:1".into(), + "Local.:1".into(), + "invalid".into(), + ], + None, + now, + ) + .unwrap() + ); + assert_eq!( + &*registry.endpoints(), + &["b:1", "shared:1", "a:1", "other:1"] + ); + assert_eq!( + registry.source_candidates["first"], + ["b:1", "shared:1", "a:1", "local:1"] + ); + assert_eq!(registry.records["shared:1"].sources.len(), 2); + assert!(!registry.records.contains_key("old:1")); + assert!(!registry.records.contains_key("local:1")); + + assert!(registry.replace_source("first", &[], None, now).unwrap()); + assert_eq!(&*registry.endpoints(), &["other:1", "shared:1"]); + assert_eq!(registry.records["shared:1"].sources.len(), 1); + registry + .replace_source("first", &["shared:1".into()], None, now) + .unwrap(); + assert_eq!(&*registry.endpoints(), &["shared:1", "other:1"]); + assert_eq!(registry.source_order, ["first", "second"]); + } + + #[test] + fn bulk_limits_count_raw_duplicates_invalid_local_and_expired_entries() { + let mut registry = CandidateRegistry::new(2).unwrap(); + let now = StdInstant::now(); + let ttl = Duration::from_secs(10); + registry.set_local_endpoints(vec!["local:1".into()]); + registry + .replace_source("existing", &["old:1".into()], Some(ttl), now) + .unwrap(); + let before = registry.clone(); + for peers in [ + vec!["Peer:1".into(), "peer.:1".into(), "peer:1".into()], + vec!["invalid".into(), "local:1".into(), "expired:1".into()], + ] { + for source in ["existing", "new"] { + assert!( + registry + .replace_source(source, &peers, Some(ttl), now) + .is_err() + ); + assert_eq!(registry, before); + let snapshot = DiscoverySnapshot::observed( + 1, + peers + .iter() + .cloned() + .map(|peer| (peer, now - ttl)) + .collect(), + ); + assert!( + registry + .replace_snapshot(source, &snapshot, Some(ttl), now) + .is_err() + ); + assert_eq!(registry, before); + } + } + } + + #[test] + fn bulk_global_limit_rolls_back_removal_reordering_and_lease_renewal() { + let mut registry = CandidateRegistry::new(3).unwrap(); + let now = StdInstant::now(); + let ttl = Duration::from_secs(10); + registry + .replace_source( + "first", + &["old:1".into(), "shared:1".into()], + Some(ttl), + now, + ) + .unwrap(); + registry + .replace_source("second", &["other:1".into(), "shared:1".into()], None, now) + .unwrap(); + let before = registry.clone(); + let peers = vec!["shared:1".into(), "new:1".into(), "overflow:1".into()]; + let later = now + ttl / 2; + let snapshot = DiscoverySnapshot::observed( + 1, + peers.iter().cloned().map(|peer| (peer, later)).collect(), + ); + for source in ["first", "new-source"] { + assert!( + registry + .replace_source(source, &peers, Some(ttl), later) + .is_err() + ); + assert_eq!(registry, before); + assert!( + registry + .replace_snapshot(source, &snapshot, Some(ttl), later) + .is_err() + ); + assert_eq!(registry, before); + } + assert!( + registry + .add("new-source", "overflow:1".into(), None, now) + .is_err() + ); + assert_eq!(registry, before); + + // Replacing an exclusive contribution releases its slot before admission. + assert!( + registry + .replace_source("first", &peers[..2], Some(ttl), later) + .unwrap() + ); + assert_eq!(&*registry.endpoints(), &["shared:1", "new:1", "other:1"]); + assert_eq!(registry.records["shared:1"].sources.len(), 2); + assert_eq!(registry.next_expiry(), Some(later + ttl)); + } + + #[test] + fn bulk_ttl_overflow_leaves_all_registry_state_unchanged() { + let mut registry = CandidateRegistry::new(3).unwrap(); + let now = StdInstant::now(); + registry + .replace_source( + "first", + &["old:1".into()], + Some(Duration::from_secs(10)), + now, + ) + .unwrap(); + registry.set_local_endpoints(vec!["local:1".into()]); + let before = registry.clone(); + for source in ["first", "new-source"] { + assert!( + registry + .replace_source(source, &["new:1".into()], Some(Duration::MAX), now) + .is_err() + ); + assert_eq!(registry, before); + assert!( + registry + .add(source, "old:1".into(), Some(Duration::MAX), now) + .is_err() + ); + assert_eq!(registry, before); + // Observations validate time arithmetic even for invalid/local entries. + for peer in ["new:1", "invalid", "local:1"] { + let snapshot = DiscoverySnapshot::observed(1, vec![(peer.into(), now)]); + assert!( + registry + .replace_snapshot(source, &snapshot, Some(Duration::MAX), now) + .is_err() + ); + assert_eq!(registry, before); + } + } + // Plain snapshots, unlike observations, never lease filtered entries. + assert!( + registry + .replace_source( + "first", + &["invalid".into(), "local:1".into()], + Some(Duration::MAX), + now + ) + .unwrap() + ); + assert!(registry.records.is_empty()); + assert!( + !registry + .replace_source("first", &[], Some(Duration::MAX), now) + .unwrap() + ); + } + + #[test] + fn bulk_observations_keep_first_live_order_and_newest_per_endpoint_lease() { + let mut registry = CandidateRegistry::new(8).unwrap(); + let now = StdInstant::now(); + let ttl = Duration::from_secs(10); + registry.set_local_endpoints(vec!["local:1".into()]); + registry + .replace_source("observed", &["old:1".into()], None, now) + .unwrap(); + registry + .replace_source("static", &["persistent:1".into(), "a:1".into()], None, now) + .unwrap(); + let snapshot = DiscoverySnapshot::observed( + 1, + vec![ + ("B:1".into(), now - ttl), + ("A.:1".into(), now - Duration::from_secs(3)), + ("b:1".into(), now - Duration::from_secs(2)), + ("a:1".into(), now - Duration::from_secs(1)), + ("A:1".into(), now - Duration::from_secs(2)), + ("local:1".into(), now), + ("invalid".into(), now), + ("expired:1".into(), now - ttl), + ], + ); + assert!( + registry + .replace_snapshot("observed", &snapshot, Some(ttl), now) + .unwrap() + ); + assert_eq!(&*registry.endpoints(), &["a:1", "b:1", "persistent:1"]); + let first_deadline = now + ttl - Duration::from_secs(2); + let last_deadline = now + ttl - Duration::from_secs(1); + assert_eq!(registry.next_expiry(), Some(first_deadline)); + assert_eq!( + registry.records["b:1"].sources["observed"].expires_at, + Some(first_deadline) + ); + assert_eq!( + registry.records["a:1"].sources["observed"].expires_at, + Some(last_deadline) + ); + assert!( + !registry + .replace_snapshot("observed", &snapshot, Some(ttl), now + ttl / 2) + .unwrap() + ); + assert_eq!(registry.next_expiry(), Some(first_deadline)); + assert!(!registry.source_unavailable("observed", true)); + assert!(!registry.expire(first_deadline - Duration::from_nanos(1))); + assert!(registry.expire(first_deadline)); + assert_eq!(&*registry.endpoints(), &["a:1", "persistent:1"]); + assert_eq!(registry.next_expiry(), Some(last_deadline)); + assert!(registry.expire(last_deadline)); + assert_eq!(&*registry.endpoints(), &["persistent:1", "a:1"]); + assert_eq!(registry.next_expiry(), None); + assert!( + !registry + .replace_snapshot("observed", &snapshot, Some(ttl), now + ttl) + .unwrap() + ); + assert_eq!(&*registry.endpoints(), &["persistent:1", "a:1"]); + } + + #[test] + fn bulk_unleased_observations_and_plain_refresh_preserve_lease_semantics() { + let mut registry = CandidateRegistry::new(3).unwrap(); + let now = StdInstant::now(); + let ttl = Duration::from_secs(10); + let peers = vec!["b:1".into(), "a:1".into()]; + registry + .replace_source("first", &peers, Some(ttl), now) + .unwrap(); + assert!( + !registry + .replace_source("first", &peers, Some(ttl), now + ttl / 2) + .unwrap() + ); + assert_eq!(registry.next_expiry(), Some(now + ttl * 3 / 2)); + let snapshot = DiscoverySnapshot::observed( + 1, + vec![ + ("B.:1".into(), now - ttl * 2), + ("a:1".into(), now - ttl), + ("b:1".into(), now), + ], + ); + assert!( + !registry + .replace_snapshot("first", &snapshot, None, now) + .unwrap() + ); + assert_eq!(&*registry.endpoints(), &["b:1", "a:1"]); + assert_eq!(registry.next_expiry(), None); + assert!(!registry.expire(now + ttl * 10)); + assert!(registry.source_unavailable("first", false)); + assert!(registry.records.is_empty()); + } + + #[test] + fn bulk_4097_candidates_rebuild_order_once_per_replacement() { + let count = 4097; + let mut registry = CandidateRegistry::new(count).unwrap(); + let now = StdInstant::now(); + let ttl = Duration::from_secs(10); + let peers = (0..count) + .map(|index| format!("peer-{index}.invalid:9000")) + .collect::>(); + assert!( + registry + .replace_source("first", &peers, Some(ttl), now) + .unwrap() + ); + assert_eq!(registry.order_rebuilds, 1); + assert_eq!(&*registry.endpoints(), &peers); + assert!( + !registry + .replace_source("second", &peers, None, now) + .unwrap() + ); + assert_eq!(registry.order_rebuilds, 2); + + let reversed = peers.iter().rev().cloned().collect::>(); + let snapshot = DiscoverySnapshot::observed( + 1, + reversed.iter().cloned().map(|peer| (peer, now)).collect(), + ); + assert!( + registry + .replace_snapshot("first", &snapshot, Some(ttl), now) + .unwrap() + ); + assert_eq!(registry.order_rebuilds, 3); + assert_eq!(&*registry.endpoints(), &reversed); + assert!( + registry + .records + .values() + .all(|record| record.sources.len() == 2) + ); + assert!( + !registry + .replace_snapshot("first", &snapshot, Some(ttl), now + ttl / 2) + .unwrap() + ); + assert_eq!(registry.order_rebuilds, 4); + assert_eq!(registry.next_expiry(), Some(now + ttl)); + assert!(registry.expire(now + ttl)); + assert_eq!(&*registry.endpoints(), &peers); + assert!( + registry + .records + .values() + .all(|record| record.sources.len() == 1) + ); + } + + #[tokio::test] + async fn identical_observation_renews_lease_but_cached_snapshot_really_expires() { + let mut registry = CandidateRegistry::new(2).unwrap(); + let now = StdInstant::now(); + let ttl = Duration::from_millis(30); + let old = DiscoverySnapshot::observed(1, vec![("peer:1".into(), now - ttl / 2)]); + let fresh = DiscoverySnapshot::observed(2, vec![("peer:1".into(), now)]); + registry + .replace_snapshot("file", &old, Some(ttl), now) + .unwrap(); + assert_eq!(registry.next_expiry(), Some(now + ttl / 2)); + assert!( + !registry + .replace_snapshot("file", &fresh, Some(ttl), now) + .unwrap() + ); + assert_eq!(registry.next_expiry(), Some(now + ttl)); + assert!( + !registry + .replace_snapshot("file", &fresh, Some(ttl), now + ttl / 2) + .unwrap() + ); + assert_eq!(registry.next_expiry(), Some(now + ttl)); + let (candidates_tx, mut candidates_rx) = watch::channel(registry.endpoints()); + let (command_tx, command_rx) = mpsc::channel(2); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let task = tokio::spawn(run_candidate_registry( + registry, + candidates_tx, + command_rx, + shutdown_rx, + )); + tokio::time::timeout(Duration::from_secs(2), candidates_rx.changed()) + .await + .unwrap() + .unwrap(); + assert!(candidates_rx.borrow_and_update().is_empty()); + // Even after actual expiry, replay/resubscription cannot resurrect it. + command_tx + .send(CandidateCommand::Snapshot { + source_id: "file".into(), + snapshot: fresh, + ttl: Some(ttl), + }) + .await + .unwrap(); + let (reply, response) = oneshot::channel(); + command_tx + .send(CandidateCommand::SetLocalEndpoints { + endpoints: Vec::new(), + reply, + }) + .await + .unwrap(); + response.await.unwrap(); + assert!(!candidates_rx.has_changed().unwrap()); + assert!(candidates_rx.borrow().is_empty()); + shutdown_tx.send_replace(true); + task.await.unwrap(); + } + + #[tokio::test] + async fn successful_identical_file_reads_keep_candidates_alive_then_invalid_file_expires() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("peers"); + tokio::fs::write(&path, "peer:1\n").await.unwrap(); + let mut config = crate::FileWatchDiscoveryConfig::new(&path); + config.poll_interval = Duration::from_millis(10); + let discovery = Arc::new(crate::FileWatchDiscovery::new(config).unwrap()); + let ttl = Duration::from_millis(150); + let provider = DiscoveryProvider::new("file", discovery.clone()).with_candidate_ttl(ttl); + let mut coordinator = + DiscoveryCoordinator::start(vec![provider], DiscoveryRuntimeConfig::new()) + .await + .unwrap(); + let mut candidates = coordinator.candidates(); + let mut observations = crate::PeerDiscovery::watch(discovery.as_ref()) + .await + .unwrap(); + let until = StdInstant::now() + ttl * 2; + tokio::time::timeout(Duration::from_secs(3), async { + loop { + let event = observations.recv().await.unwrap(); + let DiscoveryChange::Observed(snapshot) = event.change else { + panic!("missing observation"); + }; + if snapshot.observations().unwrap()[0] >= until { + break; + } + } + }) + .await + .unwrap(); + assert_eq!(&**candidates.borrow(), &["peer:1"]); + assert!(!candidates.has_changed().unwrap()); + tokio::fs::write(&path, "invalid-endpoint\n").await.unwrap(); + tokio::time::timeout(Duration::from_secs(3), candidates.changed()) + .await + .unwrap() + .unwrap(); + assert!(candidates.borrow().is_empty()); + coordinator.shutdown().await.unwrap(); + } + + #[tokio::test] + async fn coordinator_recovers_a_real_file_provider_after_worker_panic() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("peers"); + tokio::fs::write(&path, "before:9000\n").await.unwrap(); + let mut config = crate::FileWatchDiscoveryConfig::new(&path); + config.poll_interval = Duration::from_millis(10); + let discovery = Arc::new(crate::FileWatchDiscovery::new(config).unwrap()); + let source = DiscoveryProvider::new("file", discovery.clone()); + let mut coordinator = + DiscoveryCoordinator::start(vec![source], DiscoveryRuntimeConfig::new()) + .await + .unwrap(); + let mut candidates = coordinator.candidates(); + assert_eq!(&**candidates.borrow_and_update(), &["before:9000"]); + discovery.panic_on_next_observation(); + tokio::time::timeout(Duration::from_secs(3), async { + candidates.changed().await.unwrap(); + assert!(candidates.borrow_and_update().is_empty()); + // Only the coordinator may restart the provider. Do not mask a + // dead subscription by calling discover/watch from this test. + tokio::fs::write(&path, "after:9000\n").await.unwrap(); + candidates.changed().await.unwrap(); + assert_eq!(&**candidates.borrow_and_update(), &["after:9000"]); + }) + .await + .unwrap(); + coordinator.shutdown().await.unwrap(); + assert!( + crate::PeerDiscovery::watch(discovery.as_ref()) + .await + .is_err() + ); + } + + struct MutableDiscovery { + state: StdMutex<(u64, Vec)>, + events: tokio::sync::broadcast::Sender, + announced: StdMutex>, + stopped: std::sync::atomic::AtomicBool, + fail_watch: bool, + cluster: &'static str, + } + + impl MutableDiscovery { + fn new(peers: Vec) -> Self { + let (events, _) = tokio::sync::broadcast::channel(8); + Self { + state: StdMutex::new((0, peers)), + events, + announced: StdMutex::new(Vec::new()), + stopped: std::sync::atomic::AtomicBool::new(false), + fail_watch: false, + cluster: crate::DEFAULT_DISCOVERY_CLUSTER, + } + } + + fn failing() -> Self { + Self { + fail_watch: true, + ..Self::new(Vec::new()) + } + } + + fn add(&self, endpoint: &str) { + let mut state = self.state.lock().unwrap(); + state.0 += 1; + state.1.push(endpoint.to_string()); + let _ = self.events.send(crate::DiscoveryEvent { + revision: state.0, + change: DiscoveryChange::Added(endpoint.to_string()), + }); + } + + fn remove(&self, endpoint: &str) { + let mut state = self.state.lock().unwrap(); + state.0 += 1; + state.1.retain(|candidate| candidate != endpoint); + let _ = self.events.send(crate::DiscoveryEvent { + revision: state.0, + change: DiscoveryChange::Removed(endpoint.to_string()), + }); + } + } + + #[async_trait::async_trait] + impl crate::PeerDiscovery for MutableDiscovery { + fn cluster_id(&self) -> &str { + self.cluster + } + + fn announcement_support(&self) -> AnnouncementSupport { + AnnouncementSupport::Required + } + + async fn discover(&self) -> Result { + let state = self.state.lock().unwrap(); + Ok(crate::DiscoverySnapshot::new(state.0, state.1.clone())) + } + + async fn announce(&self, announcement: &PeerAnnouncement) -> Result<(), DiscoveryError> { + self.announced + .lock() + .unwrap() + .push(announcement.endpoint.clone()); + Ok(()) + } + + async fn watch(&self) -> Result { + if self.fail_watch { + return Err(DiscoveryError::Provider { + provider: "failing".to_string(), + message: "watch failed".to_string(), + retryable: false, + }); + } + let state = self.state.lock().unwrap(); + let events = self.events.subscribe(); + Ok(DiscoveryWatch::new( + crate::DiscoverySnapshot::new(state.0, state.1.clone()), + events, + )) + } + + async fn shutdown(&self) -> Result<(), DiscoveryError> { + self.stopped + .store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + } + + #[test] + fn registry_deduplicates_sources_and_removes_only_the_last_contribution() { + let mut registry = CandidateRegistry::new(4).unwrap(); + let now = StdInstant::now(); + registry + .add("one", "peer.example:9000".to_string(), None, now) + .unwrap(); + registry + .add("two", "peer.example:9000".to_string(), None, now) + .unwrap(); + + assert!(!registry.remove("one", "peer.example:9000")); + assert_eq!(&*registry.endpoints(), &["peer.example:9000"]); + assert!(registry.remove("two", "peer.example:9000")); + assert!(registry.endpoints().is_empty()); + } + + #[test] + fn registry_expiry_preserves_other_sources() { + let mut registry = CandidateRegistry::new(4).unwrap(); + let now = StdInstant::now(); + registry + .add( + "leased", + "peer.example:9000".to_string(), + Some(Duration::from_millis(5)), + now, + ) + .unwrap(); + registry + .add("static", "peer.example:9000".to_string(), None, now) + .unwrap(); + + assert!(!registry.expire(now + Duration::from_millis(10))); + assert_eq!(&*registry.endpoints(), &["peer.example:9000"]); + assert!(registry.source_unavailable("static", false)); + assert!(registry.endpoints().is_empty()); + } + + #[test] + fn registry_enforces_global_candidate_limit_atomically() { + let mut registry = CandidateRegistry::new(1).unwrap(); + registry + .replace_source( + "static", + &["one.example:9000".to_string()], + None, + StdInstant::now(), + ) + .unwrap(); + + assert!( + registry + .replace_source( + "static", + &[ + "one.example:9000".to_string(), + "two.example:9000".to_string() + ], + None, + StdInstant::now(), + ) + .is_err() + ); + assert_eq!(&*registry.endpoints(), &["one.example:9000"]); + } + + #[test] + fn registry_applies_pure_source_reordering_atomically() { + let mut registry = CandidateRegistry::new(4).unwrap(); + let now = StdInstant::now(); + registry + .replace_source( + "dynamic", + &["one.example:9000".into(), "two.example:9000".into()], + None, + now, + ) + .unwrap(); + + assert!( + registry + .replace_source( + "dynamic", + &["two.example:9000".into(), "one.example:9000".into()], + None, + now, + ) + .unwrap() + ); + assert_eq!( + &*registry.endpoints(), + &["two.example:9000", "one.example:9000"] + ); + } + + #[test] + fn removing_a_priority_contribution_publishes_the_new_source_order() { + let mut registry = CandidateRegistry::new(4).unwrap(); + let now = StdInstant::now(); + registry + .replace_source("first", &["shared.example:9000".into()], None, now) + .unwrap(); + registry + .replace_source( + "second", + &["other.example:9000".into(), "shared.example:9000".into()], + None, + now, + ) + .unwrap(); + assert_eq!( + &*registry.endpoints(), + &["shared.example:9000", "other.example:9000"] + ); + + assert!(registry.remove("first", "shared.example:9000")); + assert_eq!( + &*registry.endpoints(), + &["other.example:9000", "shared.example:9000"] + ); + } + + #[test] + fn expiry_prunes_historical_source_candidates() { + let mut registry = CandidateRegistry::new(2).unwrap(); + let now = StdInstant::now(); + registry + .add( + "leased", + "old.example:9000".into(), + Some(Duration::from_millis(1)), + now, + ) + .unwrap(); + + assert!(registry.expire(now + Duration::from_millis(2))); + assert!(registry.endpoints().is_empty()); + assert!(!registry.source_candidates.contains_key("leased")); + + registry + .add( + "leased", + "new.example:9000".into(), + Some(Duration::from_millis(1)), + now, + ) + .unwrap(); + assert_eq!(&*registry.endpoints(), &["new.example:9000"]); + } + + #[test] + fn registry_skips_invalid_snapshot_entries_without_losing_valid_candidates() { + let mut registry = CandidateRegistry::new(4).unwrap(); + + registry + .replace_source( + "static", + &[ + "not-an-endpoint".to_string(), + "Peer.Example:9000".to_string(), + "0.0.0.0:9001".to_string(), + "other.example:9002".to_string(), + ], + None, + StdInstant::now(), + ) + .unwrap(); + + assert_eq!( + &*registry.endpoints(), + &["peer.example:9000", "other.example:9002"] + ); + } + + #[test] + fn local_endpoint_is_removed_and_rejected_on_refresh() { + let mut registry = CandidateRegistry::new(4).unwrap(); + let now = StdInstant::now(); + registry + .add("static", "127.0.0.1:9000".to_string(), None, now) + .unwrap(); + + assert!(registry.set_local_endpoints(vec!["127.0.0.1:9000".to_string()])); + assert!( + !registry + .add("static", "127.0.0.1:9000".to_string(), None, now) + .unwrap() + ); + assert!(registry.endpoints().is_empty()); + } + + #[test] + fn endpoint_validation_supports_dns_and_ipv6_but_rejects_undialable_values() { + assert_eq!( + canonicalize_endpoint("Peer.Example:9000").unwrap(), + "peer.example:9000" + ); + assert_eq!(canonicalize_endpoint("[::1]:9000").unwrap(), "[::1]:9000"); + assert!(canonicalize_endpoint("0.0.0.0:9000").is_err()); + assert!(canonicalize_endpoint("224.0.0.1:9000").is_err()); + assert!(canonicalize_endpoint("255.255.255.255:9000").is_err()); + assert!(canonicalize_endpoint("[ff02::1]:9000").is_err()); + assert!(canonicalize_endpoint("[fe80::1]:9000").is_err()); + assert!(canonicalize_endpoint("peer.example:0").is_err()); + assert!(canonicalize_endpoint(" peer.example:9000").is_err()); + assert!(canonicalize_endpoint("_service.example:9000").is_err()); + assert!(canonicalize_endpoint("-peer.example:9000").is_err()); + assert_eq!( + canonicalize_endpoint("Peer.Example.:9000").unwrap(), + "peer.example:9000" + ); + } + + #[test] + fn registry_rejects_a_ttl_that_cannot_be_represented() { + let mut registry = CandidateRegistry::new(1).unwrap(); + assert!( + registry + .add( + "leased", + "peer.example:9000".to_string(), + Some(Duration::MAX), + StdInstant::now(), + ) + .is_err() + ); + assert!(registry.endpoints().is_empty()); + assert!(registry.source_order.is_empty()); + assert!(registry.source_candidates.is_empty()); + } + + #[test] + fn advertised_endpoint_uses_bound_port_and_requires_host_for_wildcard() { + let bound = "0.0.0.0:43123".parse().unwrap(); + assert_eq!(resolve_advertised_endpoint(bound, None).unwrap(), None); + assert_eq!( + resolve_advertised_endpoint(bound, Some("node.example:0")).unwrap(), + Some("node.example:43123".to_string()) + ); + assert!(resolve_advertised_endpoint(bound, Some("0.0.0.0:9000")).is_err()); + } + + #[tokio::test] + async fn coordinator_updates_an_initially_empty_snapshot_and_owns_lifecycle() { + let discovery = Arc::new(MutableDiscovery::new(Vec::new())); + let provider = DiscoveryProvider::new("dynamic", discovery.clone()); + let config = DiscoveryRuntimeConfig::new() + .with_advertised_endpoint("node.example:0") + .with_max_candidates(4); + let mut coordinator = DiscoveryCoordinator::start(vec![provider], config) + .await + .unwrap(); + let mut candidates = coordinator.candidates(); + assert!(candidates.borrow().is_empty()); + + discovery.add("Peer.Example:9000"); + tokio::time::timeout(Duration::from_secs(1), candidates.changed()) + .await + .unwrap() + .unwrap(); + assert_eq!(&**candidates.borrow_and_update(), &["peer.example:9000"]); + + discovery.remove("peer.example:9000"); + tokio::time::timeout(Duration::from_secs(1), candidates.changed()) + .await + .unwrap() + .unwrap(); + assert!(candidates.borrow_and_update().is_empty()); + + let advertised = coordinator + .configure_local_endpoint("0.0.0.0:43123".parse().unwrap()) + .await + .unwrap(); + coordinator.announce(advertised.as_deref()).await.unwrap(); + coordinator.shutdown().await.unwrap(); + + assert_eq!( + discovery.announced.lock().unwrap().as_slice(), + ["node.example:43123"] + ); + assert!(discovery.stopped.load(std::sync::atomic::Ordering::SeqCst)); + } + + #[tokio::test] + async fn coordinator_rolls_back_providers_after_partial_watch_startup() { + let first = Arc::new(MutableDiscovery::new(Vec::new())); + let failing = Arc::new(MutableDiscovery::failing()); + let providers = vec![ + DiscoveryProvider::new("first", first.clone()), + DiscoveryProvider::new("failing", failing.clone()), + ]; + + assert!( + DiscoveryCoordinator::start(providers, DiscoveryRuntimeConfig::default()) + .await + .is_err() + ); + assert!(first.stopped.load(std::sync::atomic::Ordering::SeqCst)); + assert!(failing.stopped.load(std::sync::atomic::Ordering::SeqCst)); + } + + #[tokio::test] + async fn coordinator_rejects_a_provider_from_another_cluster() { + let discovery = Arc::new(MutableDiscovery { + cluster: "other-cluster", + ..MutableDiscovery::new(Vec::new()) + }); + let result = DiscoveryCoordinator::start( + vec![DiscoveryProvider::new("foreign", discovery)], + DiscoveryRuntimeConfig::default(), + ) + .await; + + assert!(matches!( + result, + Err(DiscoveryError::InvalidConfiguration { provider, .. }) + if provider == "foreign" + )); + } +} diff --git a/crates/nx-core/src/sync_manager/manager.rs b/crates/nx-core/src/sync_manager/manager.rs index d717361..1ac92d8 100644 --- a/crates/nx-core/src/sync_manager/manager.rs +++ b/crates/nx-core/src/sync_manager/manager.rs @@ -1,7 +1,7 @@ use std::collections::HashMap; use std::sync::{Arc, atomic::AtomicU64}; -use nx_net::{Node, NodeConfig}; +use nx_net::{BootstrapServerConfig, Node, NodeConfig, PeerConnectionInfo}; use nx_store::Store as NxStore; use nx_sync::{GCounter, LwwMap, LwwRegister, NodeId, ORSet, Op, PNCounter, Rga}; use tokio::sync::{RwLock, mpsc, watch}; @@ -10,7 +10,11 @@ use tracing::{debug, info, warn}; use crate::observability::RuntimeMetrics; use crate::sync_config::SyncConfig; +use crate::{ + DEFAULT_MAX_PEER_CANDIDATES, DiscoveryProvider, DiscoveryRuntimeConfig, StaticDiscovery, +}; +use super::candidates::DiscoveryCoordinator; use super::peer::{ ConfiguredPeerConnectContext, ConfiguredPeerConnectOutcome, PeerHealth, PeerHealthState, normalize_peer_dead_after_failures, @@ -41,7 +45,7 @@ pub struct SyncHandle { rgas: Arc>>, store: Arc, metrics: Arc, - peer_node_ids: Arc>>, + active_connections: Arc>>, } impl SyncHandle { @@ -97,14 +101,26 @@ impl SyncHandle { /// Connected peers known to the sync manager, as `(addr, node_id)` pairs. pub async fn connected_peers(&self) -> Vec<(String, NodeId)> { - let peers = self.peer_node_ids.read().await; + let peers = self.active_connections.read().await; let mut peers = peers .iter() - .map(|(addr, node_id)| (addr.clone(), node_id.clone())) + .map(|(addr, connection)| (addr.clone(), connection.identity.node_id.clone())) .collect::>(); peers.sort_by(|(addr_a, _), (addr_b, _)| addr_a.cmp(addr_b)); peers } + + /// Active transport connections with handshake identity verification details. + pub async fn active_connections(&self) -> Vec { + let connections = self.active_connections.read().await; + let mut connections = connections.values().cloned().collect::>(); + connections.sort_by(|left, right| { + left.transport_addr + .cmp(&right.transport_addr) + .then_with(|| left.dialed_endpoint.cmp(&right.dialed_endpoint)) + }); + connections + } } pub struct SyncManager { @@ -114,6 +130,15 @@ pub struct SyncManager { /// SyncConfig config: SyncConfig, + /// Discovery sources whose contributions feed the shared candidate set. + discovery_providers: Vec, + + /// Discovery coordination, validation, and bound policy. + discovery_config: DiscoveryRuntimeConfig, + + /// Owner of provider watches and the live peer candidate registry. + discovery_coordinator: Option, + /// Network node. Wrapped in `Arc` so the broadcast drain task spawned /// by `start` can share ownership with the manager. node: Option>, @@ -154,13 +179,13 @@ pub struct SyncManager { /// Monotonic sequence used to retain recent durable operation-log entries. op_log_next_sequence: Arc, - /// Health state for configured peers, keyed by configured address. + /// Health state for current discovery candidates, keyed by dial endpoint. peer_health: Arc>>, - /// Connected configured peer NodeIds, keyed by configured address. - peer_node_ids: Arc>>, + /// Active connections, keyed by the address used by the network node. + active_connections: Arc>>, - /// Last received OpId per peer NodeId, used for incremental anti-entropy pulls. + /// Last received OpId per peer NodeId, for observation only, not a causal frontier. anti_entropy_watermarks: Arc>>, /// Channel to send Ops to broadcast. @@ -186,27 +211,52 @@ pub struct SyncManager { } impl SyncManager { - /// Create a SyncManager, panicking if the persisted schema is invalid. + /// Create a SyncManager, panicking if configuration or persistence is invalid. /// - /// Runtime integrations should prefer [`Self::try_new`] so schema errors - /// can be reported without terminating the process. + /// Runtime integrations should prefer [`Self::try_new`] so configuration and + /// schema errors can be reported without terminating the process. pub fn new( node_id: NodeId, config: SyncConfig, store: Arc, metrics: Arc, ) -> Self { - Self::try_new(node_id, config, store, metrics) - .expect("failed to initialize SyncManager persistence") + Self::try_new(node_id, config, store, metrics).expect("failed to initialize SyncManager") } - /// Create a SyncManager after validating all managed persistence schemas. + /// Create a SyncManager after validating configuration and managed persistence schemas. pub fn try_new( node_id: NodeId, config: SyncConfig, store: Arc, metrics: Arc, ) -> anyhow::Result { + let static_discovery = Arc::new(StaticDiscovery::new(config.peers.clone())); + // Explicit peers were accepted as a finite caller-owned list before the + // discovery coordinator existed. Keep that compatibility while retaining + // the configured bound for every dynamic-discovery construction path. + let discovery_config = DiscoveryRuntimeConfig::default() + .with_max_candidates(DEFAULT_MAX_PEER_CANDIDATES.max(config.peers.len())); + Self::try_new_with_discovery( + node_id, + config, + store, + metrics, + vec![DiscoveryProvider::new("static", static_discovery)], + discovery_config, + ) + } + + /// Create a manager with explicitly owned discovery sources and policy. + pub fn try_new_with_discovery( + node_id: NodeId, + config: SyncConfig, + store: Arc, + metrics: Arc, + discovery_providers: Vec, + discovery_config: DiscoveryRuntimeConfig, + ) -> anyhow::Result { + config.validate()?; ensure_sync_schema(&store)?; let (op_tx, op_rx) = mpsc::channel(config.queued_ops_limit.max(1)); @@ -244,6 +294,9 @@ impl SyncManager { Ok(Self { node_id, config, + discovery_providers, + discovery_config, + discovery_coordinator: None, node: None, counters, pncounters, @@ -258,7 +311,7 @@ impl SyncManager { op_log: Arc::new(RwLock::new(op_log)), op_log_next_sequence: Arc::new(AtomicU64::new(op_log_next_sequence)), peer_health: Arc::new(RwLock::new(peer_health)), - peer_node_ids: Arc::new(RwLock::new(HashMap::new())), + active_connections: Arc::new(RwLock::new(HashMap::new())), anti_entropy_watermarks: Arc::new(RwLock::new(HashMap::new())), op_tx, op_rx: Some(op_rx), @@ -298,12 +351,16 @@ impl SyncManager { rgas: Arc::clone(&self.rgas), store: Arc::clone(&self.store), metrics: Arc::clone(&self.metrics), - peer_node_ids: Arc::clone(&self.peer_node_ids), + active_connections: Arc::clone(&self.active_connections), } } - /// Start networking: bind the listener, dial initial peers, spawn the inbound event loop and the outbound broadcast drain loop. + /// Bind the listener and start discovery, replication, and reconnect tasks. + /// + /// Initial peers are dialed in the background. Success means local services + /// are started, not that a peer is connected or replication has settled. pub async fn start(&mut self) -> anyhow::Result<()> { + self.config.validate()?; let listen_addr = match &self.config.listen_addr { Some(addr) => addr.clone(), None => { @@ -312,43 +369,89 @@ impl SyncManager { } }; - // Build the network node. + if self.node.is_some() || self.op_rx.is_none() { + anyhow::bail!("sync manager is already started"); + } + + // Reject local configuration before acquiring provider watches or tasks. + // The reconnect loop consumes live candidates, not NodeConfig::initial_peers. let mut node_config = NodeConfig::new(self.node_id.clone(), &listen_addr) - .with_peers(self.config.peers.clone()) .with_max_peers(self.config.max_peers) .with_max_message_size(self.config.max_message_size) .with_socket_timeout(self.config.socket_timeout) .with_serialization_format(self.config.serialization_format) .with_event_channel_capacity(self.config.queued_ops_limit.max(1)); - + let bootstrap_server = BootstrapServerConfig::new(self.discovery_config.cluster_id())? + .with_max_cached_candidates(self.discovery_config.max_candidates())? + .with_max_response_candidates( + self.discovery_config + .max_candidates() + .min(nx_net::MAX_BOOTSTRAP_RESPONSE_CAPACITY), + )?; if let Some(tls) = self.config.tls.clone() { node_config = node_config.with_tls(tls); } - let mut node = Node::new(node_config); - let mut event_rx = node.take_event_receiver().unwrap(); - - node.start_listener().await?; + let mut node = Node::try_new_with_bootstrap_server(node_config, bootstrap_server)?; + let Some(mut event_rx) = node.take_event_receiver() else { + anyhow::bail!("network event receiver is unavailable"); + }; - // Connect to initial peers. - let peer_dead_after_failures = - normalize_peer_dead_after_failures(self.config.peer_dead_after_failures); - let connect_context = ConfiguredPeerConnectContext { - node: &node, - max_peers: self.config.max_peers, - peer_dead_after_failures, - metrics: &self.metrics, - peer_health: &self.peer_health, + // Provider watches are acquired before binding so discovery startup is + // atomic with respect to network resources. All later failures roll back. + let mut discovery_coordinator = DiscoveryCoordinator::start( + self.discovery_providers.clone(), + self.discovery_config.clone(), + ) + .await?; + let candidates_rx = discovery_coordinator.candidates(); + + let bound_addr = match node.start_listener().await { + Ok(bound_addr) => bound_addr, + Err(error) => { + rollback_discovery(&mut discovery_coordinator).await; + return Err(error.into()); + } }; - for peer_addr in &self.config.peers { - if matches!( - try_connect_configured_peer(&connect_context, peer_addr).await, - ConfiguredPeerConnectOutcome::SlotLimitReached - ) { - break; + let advertised_endpoint = match discovery_coordinator + .configure_local_endpoint(bound_addr) + .await + { + Ok(endpoint) => endpoint, + Err(error) => { + node.shutdown().await; + rollback_discovery(&mut discovery_coordinator).await; + return Err(error.into()); } + }; + if let Some(endpoint) = &advertised_endpoint + && let Err(error) = node.announce_bootstrap_endpoint(endpoint.clone()) + { + node.shutdown().await; + rollback_discovery(&mut discovery_coordinator).await; + return Err(error.into()); + } + if let Err(error) = discovery_coordinator + .announce(advertised_endpoint.as_deref()) + .await + { + node.shutdown().await; + rollback_discovery(&mut discovery_coordinator).await; + return Err(error.into()); } + let initial_candidates = Arc::clone(&candidates_rx.borrow()); + *self.peer_health.write().await = initial_candidates + .iter() + .map(|peer| (peer.clone(), PeerHealth::default())) + .collect(); + + let Some(op_rx) = self.op_rx.take() else { + node.shutdown().await; + rollback_discovery(&mut discovery_coordinator).await; + anyhow::bail!("sync manager is already started"); + }; + // Move the node into an Arc so it can be shared between the manager and the broadcast drain task. let node = Arc::new(node); self.node = Some(Arc::clone(&node)); @@ -370,7 +473,7 @@ impl SyncManager { metrics: Arc::clone(&self.metrics), node: Arc::clone(&node), peer_health: Arc::clone(&self.peer_health), - peer_node_ids: Arc::clone(&self.peer_node_ids), + active_connections: Arc::clone(&self.active_connections), anti_entropy_watermarks: Arc::clone(&self.anti_entropy_watermarks), peer_dead_after_failures: normalize_peer_dead_after_failures( self.config.peer_dead_after_failures, @@ -399,10 +502,6 @@ impl SyncManager { })); // Outbound loop: drain locally-produced ops into the network. - let op_rx = self - .op_rx - .take() - .expect("op_rx already taken: SyncManager::start called twice?"); self.broadcast_task = Some(spawn_broadcast_loop( BroadcastLoopContext { node: Arc::clone(&node), @@ -420,7 +519,7 @@ impl SyncManager { self.reconnect_task = spawn_reconnect_loop(ReconnectLoopContext { node: Arc::clone(&node), - peers: self.config.peers.clone(), + candidates_rx, max_peers: self.config.max_peers, initial_delay: self.config.reconnect_initial_delay, max_delay: self.config.reconnect_max_delay, @@ -432,12 +531,13 @@ impl SyncManager { self.anti_entropy_task = spawn_anti_entropy_loop(AntiEntropyLoopContext { node: Arc::clone(&node), - peers: self.config.peers.clone(), interval: self.config.anti_entropy_interval, shutdown_rx: self.shutdown_tx.subscribe(), metrics: Arc::clone(&self.metrics), }); + self.discovery_coordinator = Some(discovery_coordinator); + Ok(()) } @@ -450,7 +550,7 @@ impl SyncManager { Ok(()) } - /// Retry connecting to the peers configured at startup. + /// Retry connecting to the current discovery candidates. pub async fn reconnect_configured_peers(&self) { let Some(node) = self.node.as_ref() else { return; @@ -464,7 +564,12 @@ impl SyncManager { metrics: &self.metrics, peer_health: &self.peer_health, }; - for peer_addr in &self.config.peers { + let Some(coordinator) = self.discovery_coordinator.as_ref() else { + return; + }; + let candidates_rx = coordinator.candidates(); + let candidates = Arc::clone(&candidates_rx.borrow()); + for peer_addr in candidates.iter() { if matches!( try_connect_configured_peer(&connect_context, peer_addr).await, ConfiguredPeerConnectOutcome::SlotLimitReached @@ -474,12 +579,21 @@ impl SyncManager { } } - /// Returns the current health state of a configured peer. + /// Returns the current health state of a discovery candidate. pub async fn peer_health_state(&self, addr: &str) -> Option { let peer_health = self.peer_health.read().await; peer_health.get(addr).map(|health| health.state) } + /// Returns the current ordered, deduplicated discovery candidate snapshot. + pub fn peer_candidates(&self) -> Vec { + let Some(coordinator) = self.discovery_coordinator.as_ref() else { + return Vec::new(); + }; + let candidates = coordinator.candidates(); + candidates.borrow().as_ref().clone() + } + /// Returns the number of connected peers, or zero before networking starts. pub async fn connected_peer_count(&self) -> usize { let Some(node) = self.node.as_ref() else { @@ -527,6 +641,11 @@ impl SyncManager { /// Gracefully stop sync tasks and close network connections. pub async fn shutdown(&mut self) -> anyhow::Result<()> { let _ = self.shutdown_tx.send(true); + let mut discovery_coordinator = self.discovery_coordinator.take(); + if let Some(coordinator) = discovery_coordinator.as_ref() { + coordinator.request_shutdown(); + } + let mut discovery_error = None; if let Some(task) = self.broadcast_task.take() && let Err(e) = task.await @@ -546,6 +665,13 @@ impl SyncManager { warn!(error = %e, "anti-entropy task failed during shutdown"); } + if let Some(mut coordinator) = discovery_coordinator.take() + && let Err(error) = coordinator.shutdown().await + { + warn!(error = %error, "discovery shutdown failed"); + discovery_error = Some(error); + } + if let Some(node) = self.node.as_ref() { node.shutdown().await; } @@ -558,7 +684,13 @@ impl SyncManager { } info!("sync manager shut down"); - Ok(()) + discovery_error.map_or(Ok(()), |error| Err(error.into())) + } +} + +async fn rollback_discovery(coordinator: &mut DiscoveryCoordinator) { + if let Err(error) = coordinator.shutdown().await { + warn!(error = %error, "discovery rollback failed"); } } diff --git a/crates/nx-core/src/sync_manager/mod.rs b/crates/nx-core/src/sync_manager/mod.rs index dae1cad..f561d96 100644 --- a/crates/nx-core/src/sync_manager/mod.rs +++ b/crates/nx-core/src/sync_manager/mod.rs @@ -1,4 +1,6 @@ mod apply; +mod candidates; +pub(crate) use candidates::canonicalize_endpoint; mod manager; mod migration; mod peer; diff --git a/crates/nx-core/src/sync_manager/peer.rs b/crates/nx-core/src/sync_manager/peer.rs index bb5ac4b..bdcab3e 100644 --- a/crates/nx-core/src/sync_manager/peer.rs +++ b/crates/nx-core/src/sync_manager/peer.rs @@ -52,17 +52,34 @@ impl PeerReconnectState { self.stopped = false; } - pub(super) fn record_failure(&mut self, max_delay: Duration, now: StdInstant) -> Duration { + pub(super) fn record_failure( + &mut self, + max_delay: Duration, + now: StdInstant, + ) -> Option { let attempt_delay = self.delay; - self.next_attempt_at = now + attempt_delay; + self.schedule_retry(attempt_delay, now)?; self.delay = next_reconnect_delay(attempt_delay, max_delay); - attempt_delay + Some(attempt_delay) } - pub(super) fn record_retry_after(&mut self, delay: Duration, now: StdInstant) -> Duration { + pub(super) fn record_retry_after( + &mut self, + delay: Duration, + now: StdInstant, + ) -> Option { let delay = normalize_reconnect_delay(delay); - self.next_attempt_at = now.checked_add(delay).unwrap_or(now); - delay + self.schedule_retry(delay, now) + } + + fn schedule_retry(&mut self, delay: Duration, now: StdInstant) -> Option { + let Some(deadline) = now.checked_add(delay) else { + // An unrepresentable deadline must never become an immediate retry. + self.stop(); + return None; + }; + self.next_attempt_at = deadline; + Some(delay) } pub(super) fn stop(&mut self) { @@ -197,7 +214,7 @@ mod tests { PeerReconnectState::new("peer-a".to_string(), Duration::from_millis(500), now); let first_delay = state.record_failure(Duration::from_secs(5), now); - assert_eq!(first_delay, Duration::from_millis(500)); + assert_eq!(first_delay, Some(Duration::from_millis(500))); assert_eq!(state.delay, Duration::from_secs(1)); assert_eq!(state.next_attempt_at, now + Duration::from_millis(500)); assert!(!state.stopped); @@ -215,7 +232,9 @@ mod tests { let mut state = PeerReconnectState::new("peer-a".to_string(), Duration::from_millis(500), started_at); - state.record_failure(Duration::from_secs(5), failed_at); + state + .record_failure(Duration::from_secs(5), failed_at) + .unwrap(); assert_eq!( state.next_attempt_at, @@ -231,7 +250,7 @@ mod tests { let delay = state.record_retry_after(Duration::from_secs(3), now); - assert_eq!(delay, Duration::from_secs(3)); + assert_eq!(delay, Some(Duration::from_secs(3))); assert_eq!(state.delay, Duration::from_millis(500)); assert_eq!(state.next_attempt_at, now + Duration::from_secs(3)); } @@ -244,8 +263,17 @@ mod tests { let delay = state.record_retry_after(Duration::MAX, now); - assert_eq!(delay, Duration::MAX); - assert_eq!(state.next_attempt_at, now); + assert_eq!(delay, None); + assert!(state.stopped); + } + + #[test] + fn peer_reconnect_state_stops_on_unrepresentable_failure_deadline() { + let now = StdInstant::now(); + let mut state = PeerReconnectState::new("peer-a".to_string(), Duration::MAX, now); + + assert_eq!(state.record_failure(Duration::MAX, now), None); + assert!(state.stopped); } #[test] diff --git a/crates/nx-core/src/sync_manager/replication.rs b/crates/nx-core/src/sync_manager/replication.rs index 7b95e69..cd2bd1c 100644 --- a/crates/nx-core/src/sync_manager/replication.rs +++ b/crates/nx-core/src/sync_manager/replication.rs @@ -1,3 +1,4 @@ +use std::collections::{HashMap, HashSet}; use std::sync::{ Arc, atomic::{AtomicU64, Ordering}, @@ -141,10 +142,16 @@ fn bounded_retry_after(delay: Duration, max_delay: Duration) -> Duration { normalize_reconnect_delay(delay).min(max_delay) } +async fn wait_for_shutdown(shutdown_rx: &mut watch::Receiver) { + // Do not return watch::Ref from a select branch: its non-Send guard can + // otherwise be retained across an await in another branch's handler. + let _ = shutdown_rx.wait_for(|shutdown| *shutdown).await; +} + pub(super) fn spawn_reconnect_loop(context: ReconnectLoopContext) -> Option> { let ReconnectLoopContext { node, - peers, + mut candidates_rx, max_peers, initial_delay, max_delay, @@ -154,21 +161,24 @@ pub(super) fn spawn_reconnect_loop(context: ReconnectLoopContext) -> Option>(); + let mut state = Vec::new(); + let initial_candidates = Arc::clone(&candidates_rx.borrow()); + reconcile_reconnect_candidates( + &mut state, + initial_candidates.as_ref(), + initial_delay, + &peer_health, + ) + .await; - loop { + 'reconnect: loop { + if *shutdown_rx.borrow() { + break; + } let mut sleep_for: Option = None; let now = StdInstant::now(); let connect_context = ConfiguredPeerConnectContext { @@ -199,7 +209,14 @@ pub(super) fn spawn_reconnect_loop(context: ReconnectLoopContext) -> Option break 'reconnect, + outcome = try_connect_configured_peer(&connect_context, peer_addr) => outcome, + }; + match outcome { ConfiguredPeerConnectOutcome::Connected => { info!(peer = %peer_addr, "reconnected configured peer"); peer.reset(initial_delay, StdInstant::now()); @@ -211,16 +228,25 @@ pub(super) fn spawn_reconnect_loop(context: ReconnectLoopContext) -> Option { - let attempt_delay = peer.record_failure(max_delay, StdInstant::now()); + let Some(attempt_delay) = peer.record_failure(max_delay, StdInstant::now()) + else { + metrics.record_sync_error(); + warn!(peer = %peer.addr, "stopping reconnect: backoff deadline overflow"); + continue; + }; sleep_for = Some( sleep_for.map_or(attempt_delay, |current| current.min(attempt_delay)), ); } ConfiguredPeerConnectOutcome::RetryAfter(delay) => { - let retry_after = peer.record_retry_after( + let Some(retry_after) = peer.record_retry_after( bounded_retry_after(delay, max_delay), StdInstant::now(), - ); + ) else { + metrics.record_sync_error(); + warn!(peer = %peer.addr, "stopping reconnect: retry-after deadline overflow"); + continue; + }; sleep_for = Some(sleep_for.map_or(retry_after, |current| current.min(retry_after))); } @@ -234,15 +260,30 @@ pub(super) fn spawn_reconnect_loop(context: ReconnectLoopContext) -> Option { - if *shutdown_rx.borrow() { - debug!("reconnect loop shutdown requested"); + _ = wait_for_shutdown(&mut shutdown_rx) => { + debug!("reconnect loop shutdown requested"); + break; + } + changed = candidates_rx.changed() => { + if changed.is_err() { break; } + let candidates = Arc::clone(&candidates_rx.borrow_and_update()); + reconcile_reconnect_candidates( + &mut state, + candidates.as_ref(), + initial_delay, + &peer_health, + ).await; } - _ = tokio::time::sleep(sleep_for) => {} + _ = tokio::time::sleep_until(deadline) => {} } } debug!("reconnect loop terminated"); @@ -252,50 +293,99 @@ pub(super) fn spawn_reconnect_loop(context: ReconnectLoopContext) -> Option Option> { let AntiEntropyLoopContext { node, - peers, interval, mut shutdown_rx, metrics, } = context; - if peers.is_empty() { - return None; - } - Some(tokio::spawn(async move { let interval = normalize_anti_entropy_interval(interval); + // Keep the cadence independent of discovery churn and skip missed ticks + // rather than issuing bursts after a slow transport write. Recheck every + // deadline: construction-time validation cannot guarantee future additions. + let mut previous = tokio::time::Instant::now(); loop { + let Some(deadline) = + checked_anti_entropy_deadline(previous, interval, tokio::time::Instant::now()) + else { + metrics.record_sync_error(); + warn!("stopping anti-entropy loop: cadence deadline overflow"); + break; + }; + previous = deadline; tokio::select! { - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - debug!("anti-entropy loop shutdown requested"); - break; - } + biased; + _ = wait_for_shutdown(&mut shutdown_rx) => { + debug!("anti-entropy loop shutdown requested"); + break; } - _ = tokio::time::sleep(interval) => { - for peer in &peers { - if !node.is_connected_addr(peer).await { - continue; - } - + _ = async { + tokio::time::sleep_until(deadline).await; + // These are Node's send-address keys, including inbound + // connections and peers no longer present in discovery. + for (peer, _) in node.connected_peers().await { // A single "last seen OpId" is not a safe causal frontier: a peer can // receive a newer op while an older broadcast was dropped. Until the // protocol has contiguous/causal metadata, anti-entropy pulls the bounded // op-log and relies on OpId deduplication on the receiver. - if let Err(e) = node.send_pull_since_to_addr(peer, None).await { + if let Err(e) = node.send_pull_since_to_addr(&peer, None).await { metrics.record_sync_error(); debug!(peer = %peer, error = %e, "anti-entropy pull failed"); } else { debug!(peer = %peer, "anti-entropy pull requested"); } } - } + } => {} } } debug!("anti-entropy loop terminated"); })) } +fn checked_anti_entropy_deadline( + previous: tokio::time::Instant, + interval: Duration, + now: tokio::time::Instant, +) -> Option { + let interval = normalize_anti_entropy_interval(interval); + let next = previous.checked_add(interval)?; + if next > now { + return Some(next); + } + // Preserve the original phase while skipping ticks missed during transport I/O. + let remainder = now.duration_since(previous).as_nanos() % interval.as_nanos(); + let remainder = Duration::new( + (remainder / 1_000_000_000) as u64, + (remainder % 1_000_000_000) as u32, + ); + now.checked_add(interval - remainder) +} + +async fn reconcile_reconnect_candidates( + state: &mut Vec, + candidates: &[String], + initial_delay: Duration, + peer_health: &Arc>>, +) { + let retained = candidates.iter().collect::>(); + let mut existing = state + .drain(..) + .map(|peer| (peer.addr.clone(), peer)) + .collect::>(); + let now = StdInstant::now(); + state.extend(candidates.iter().map(|addr| { + existing + .remove(addr) + .unwrap_or_else(|| PeerReconnectState::new(addr.clone(), initial_delay, now)) + })); + + let mut health = peer_health.write().await; + health.retain(|addr, _| retained.contains(addr)); + for candidate in candidates { + health.entry(candidate.clone()).or_default(); + } +} + pub(super) fn normalize_anti_entropy_interval(interval: Duration) -> Duration { interval.max(Duration::from_millis(1)) } @@ -552,11 +642,17 @@ pub(super) async fn handle_node_event(event: NodeEvent, context: &NodeEventConte peers_connected, } => { mark_known_peer_success(&context.peer_health, &addr).await; - context - .peer_node_ids - .write() - .await - .insert(addr.clone(), node_id.clone()); + if let Some(connection) = context.node.connection_info(&addr).await { + if connection.identity.node_id == node_id { + context + .active_connections + .write() + .await + .insert(addr.clone(), connection); + } else { + warn!(peer = %node_id, addr = %addr, "ignored inconsistent connection identity"); + } + } context.metrics.record_peer_connect(); context.metrics.set_peers_connected(peers_connected); info!(peer = %node_id, addr = %addr, "peer connected"); @@ -572,7 +668,7 @@ pub(super) async fn handle_node_event(event: NodeEvent, context: &NodeEventConte context.peer_dead_after_failures, ) .await; - context.peer_node_ids.write().await.remove(&addr); + context.active_connections.write().await.remove(&addr); context.metrics.record_peer_disconnect(); context.metrics.set_peers_connected(peers_connected); info!(peer = %node_id, addr = %addr, "peer disconnected"); @@ -610,6 +706,122 @@ mod tests { Arc::new(RuntimeMetrics::default()) } + #[test] + fn anti_entropy_deadlines_are_checked_and_skip_missed_ticks_without_bursts() { + let now = tokio::time::Instant::now(); + let interval = Duration::from_millis(10); + assert_eq!(checked_anti_entropy_deadline(now, Duration::MAX, now), None); + assert_eq!( + checked_anti_entropy_deadline(now, interval, now), + now.checked_add(interval) + ); + assert_eq!( + checked_anti_entropy_deadline(now, interval, now + Duration::from_millis(35)), + now.checked_add(Duration::from_millis(40)), + ); + assert_eq!( + checked_anti_entropy_deadline(now, Duration::ZERO, now), + now.checked_add(Duration::from_millis(1)), + ); + } + + #[tokio::test] + async fn anti_entropy_task_exits_without_panicking_on_deadline_overflow() { + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + let task = spawn_anti_entropy_loop(AntiEntropyLoopContext { + node: Arc::new( + Node::try_new(NodeConfig::new(NodeId::generate(), "127.0.0.1:0")).unwrap(), + ), + interval: Duration::MAX, + shutdown_rx, + metrics: metrics(), + }) + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn reconnect_task_exits_without_panicking_on_sleep_deadline_overflow() { + let (_shutdown_tx, shutdown_rx) = watch::channel(false); + let (_candidates_tx, candidates_rx) = watch::channel(Arc::new(Vec::new())); + let task = spawn_reconnect_loop(ReconnectLoopContext { + node: Arc::new( + Node::try_new(NodeConfig::new(NodeId::generate(), "127.0.0.1:0")).unwrap(), + ), + candidates_rx, + max_peers: 0, + initial_delay: Duration::MAX, + max_delay: Duration::MAX, + peer_dead_after_failures: 1, + shutdown_rx, + metrics: metrics(), + peer_health: Arc::new(RwLock::new(HashMap::new())), + }) + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), task) + .await + .unwrap() + .unwrap(); + } + + #[tokio::test] + async fn removed_candidates_are_deleted_from_reconnect_state_and_health() { + let now = StdInstant::now(); + let mut state = vec![PeerReconnectState::new( + "peer.example:9000".to_string(), + Duration::from_millis(10), + now, + )]; + let peer_health = Arc::new(RwLock::new(HashMap::from([( + "peer.example:9000".to_string(), + PeerHealth::default(), + )]))); + + reconcile_reconnect_candidates(&mut state, &[], Duration::from_millis(10), &peer_health) + .await; + + assert!(state.is_empty()); + assert!(peer_health.read().await.is_empty()); + } + + #[tokio::test] + async fn candidate_reordering_preserves_backoff_state() { + let now = StdInstant::now(); + let mut first = PeerReconnectState::new( + "one.example:9000".to_string(), + Duration::from_millis(10), + now, + ); + first.record_failure(Duration::from_secs(1), now).unwrap(); + let first_deadline = first.next_attempt_at; + let first_delay = first.delay; + let mut state = vec![ + first, + PeerReconnectState::new( + "two.example:9000".to_string(), + Duration::from_millis(10), + now, + ), + ]; + let peer_health = Arc::new(RwLock::new(HashMap::new())); + + reconcile_reconnect_candidates( + &mut state, + &["two.example:9000".into(), "one.example:9000".into()], + Duration::from_millis(10), + &peer_health, + ) + .await; + + assert_eq!(state[0].addr, "two.example:9000"); + assert_eq!(state[1].addr, "one.example:9000"); + assert_eq!(state[1].next_attempt_at, first_deadline); + assert_eq!(state[1].delay, first_delay); + } + fn test_event_context( counters: Arc>>, seen_ops: Arc>, @@ -637,7 +849,7 @@ mod tests { "127.0.0.1:0", ))), peer_health, - peer_node_ids: Arc::new(RwLock::new(HashMap::new())), + active_connections: Arc::new(RwLock::new(HashMap::new())), anti_entropy_watermarks: Arc::new(RwLock::new(HashMap::new())), peer_dead_after_failures: 2, } diff --git a/crates/nx-core/src/sync_manager/tests/mod.rs b/crates/nx-core/src/sync_manager/tests/mod.rs index 3bcb4c1..8eda53e 100644 --- a/crates/nx-core/src/sync_manager/tests/mod.rs +++ b/crates/nx-core/src/sync_manager/tests/mod.rs @@ -1,8 +1,9 @@ use super::*; use crate::runtime::{Runtime, RuntimeConfig}; use crate::sync_manager::{apply::*, peer::*, replication::*, storage::*}; -use nx_net::NodeEvent; +use nx_net::{ConnectionDirection, NodeEvent, PeerIdentityVerification}; use nx_sync::OpKind; +use std::sync::Mutex as StdMutex; use std::time::Instant as StdInstant; use std::time::{SystemTime, UNIX_EPOCH}; use tokio::time::{Duration, Instant, sleep}; @@ -12,6 +13,68 @@ mod support; use support::*; +struct TestDynamicDiscovery { + state: StdMutex<(u64, Vec)>, + events: tokio::sync::broadcast::Sender, +} + +impl TestDynamicDiscovery { + fn empty() -> Self { + let (events, _) = tokio::sync::broadcast::channel(8); + Self { + state: StdMutex::new((0, Vec::new())), + events, + } + } + + fn add(&self, endpoint: String) { + let mut state = self.state.lock().unwrap(); + state.0 += 1; + state.1.push(endpoint.clone()); + let _ = self.events.send(crate::DiscoveryEvent { + revision: state.0, + change: crate::DiscoveryChange::Added(endpoint), + }); + } + + fn remove(&self, endpoint: &str) { + let mut state = self.state.lock().unwrap(); + state.0 += 1; + state.1.retain(|candidate| candidate != endpoint); + let _ = self.events.send(crate::DiscoveryEvent { + revision: state.0, + change: crate::DiscoveryChange::Removed(endpoint.to_string()), + }); + } +} + +#[async_trait::async_trait] +impl crate::PeerDiscovery for TestDynamicDiscovery { + async fn discover(&self) -> Result { + let state = self.state.lock().unwrap(); + Ok(crate::DiscoverySnapshot::new(state.0, state.1.clone())) + } + + async fn announce( + &self, + _announcement: &crate::PeerAnnouncement, + ) -> Result<(), crate::DiscoveryError> { + Err(crate::DiscoveryError::Unsupported { + provider: "test-dynamic".to_string(), + operation: "announcement", + }) + } + + async fn watch(&self) -> Result { + let state = self.state.lock().unwrap(); + let events = self.events.subscribe(); + Ok(crate::DiscoveryWatch::new( + crate::DiscoverySnapshot::new(state.0, state.1.clone()), + events, + )) + } +} + #[test] fn crdt_store_keys_roundtrip_through_generic_namespace_helpers() { let materialized = crdt_store_key( @@ -240,7 +303,7 @@ fn peer_reconnect_state_tracks_next_attempt_per_peer() { let mut state = PeerReconnectState::new("peer-a".to_string(), Duration::from_millis(500), now); let first_delay = state.record_failure(Duration::from_secs(5), now); - assert_eq!(first_delay, Duration::from_millis(500)); + assert_eq!(first_delay, Some(Duration::from_millis(500))); assert_eq!(state.delay, Duration::from_secs(1)); assert_eq!(state.next_attempt_at, now + Duration::from_millis(500)); @@ -256,7 +319,9 @@ fn peer_reconnect_state_schedules_backoff_from_failure_time() { let mut state = PeerReconnectState::new("peer-a".to_string(), Duration::from_millis(500), started_at); - state.record_failure(Duration::from_secs(5), failed_at); + state + .record_failure(Duration::from_secs(5), failed_at) + .unwrap(); assert_eq!( state.next_attempt_at, @@ -1462,6 +1527,197 @@ fn manager_rejects_corrupted_durable_crdt_state() { } } +#[tokio::test] +async fn static_peer_lists_keep_their_historical_finite_size_at_startup() { + let peers = (0..=nx_net::MAX_BOOTSTRAP_RESPONSE_CAPACITY) + .map(|index| format!("peer-{index}.invalid:9000")) + .collect::>(); + // Disable outbound admission: this tests real candidate retention, not DNS/dials. + let mut config = SyncConfig::new() + .with_listen_addr("127.0.0.1:0") + .with_max_peers(0); + config.peers = peers.clone(); + + let mut manager = + SyncManager::try_new(NodeId::new("local-node"), config, temp_store(), metrics()).unwrap(); + + assert_eq!(manager.discovery_config.max_candidates(), peers.len()); + manager.start().await.unwrap(); + assert_eq!(manager.peer_candidates(), peers); + manager.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn dynamic_provider_modes_accept_aggregate_capacity_above_bootstrap_response_limit() { + let peers = vec![ + "peer-a.invalid:9000".to_string(), + "peer-b.invalid:9000".to_string(), + ]; + // Controlled snapshots isolate the manager contract from platform discovery I/O. + for mode in ["mdns", "dns-srv", "file"] { + let discovery = Arc::new(TestDynamicDiscovery::empty()); + discovery.state.lock().unwrap().1 = peers.clone(); + let mut manager = SyncManager::try_new_with_discovery( + NodeId::new("local-node"), + SyncConfig::new() + .with_listen_addr("127.0.0.1:0") + .with_max_peers(0), + temp_store(), + metrics(), + vec![DiscoveryProvider::new(mode, discovery)], + DiscoveryRuntimeConfig::default() + .with_max_candidates(nx_net::MAX_BOOTSTRAP_RESPONSE_CAPACITY + 1), + ) + .unwrap(); + + manager.start().await.unwrap(); + assert_eq!(manager.peer_candidates(), peers, "{mode}"); + manager.shutdown().await.unwrap(); + } +} + +#[tokio::test] +async fn bootstrap_responses_are_capped_without_reducing_the_aggregate_cache() { + use nx_net::{BootstrapClient, BootstrapClientConfig, BootstrapRequest}; + let cap = nx_net::MAX_BOOTSTRAP_RESPONSE_CAPACITY; + let discovery_config = DiscoveryRuntimeConfig::default().with_max_candidates(cap + 1); + let cluster = discovery_config.cluster_id().to_string(); + let addr = free_addr(); + let mut manager = SyncManager::try_new_with_discovery( + NodeId::new("bootstrap-seed"), + SyncConfig::new().with_listen_addr(&addr), + temp_store(), + metrics(), + vec![], + discovery_config, + ) + .unwrap(); + manager.start().await.unwrap(); + + let client = |index: usize| { + let mut config = + BootstrapClientConfig::new(NodeId::new(format!("bootstrap-client-{index}"))); + config.max_response_candidates = cap; + BootstrapClient::new(config).unwrap() + }; + // Fill the actual server cache through one-shot handshakes, requesting only + // one result while filling so the regression does not transfer quadratic data. + for index in 0..=cap { + client(index) + .query( + &addr, + BootstrapRequest::new(&cluster, 1) + .with_advertised_endpoint(format!("peer-{index}.invalid:9000")), + ) + .await + .unwrap(); + } + let response = client(cap + 1) + .query(&addr, BootstrapRequest::new(&cluster, cap)) + .await + .unwrap(); + assert_eq!(response.endpoints.len(), cap); + assert!( + !response + .endpoints + .contains(&format!("peer-{cap}.invalid:9000")) + ); + + // Free two earlier entries. The last contribution must emerge; a cache + // incorrectly clamped to 4096 would have discarded it during admission. + client(0) + .query(&addr, BootstrapRequest::new(&cluster, 1)) + .await + .unwrap(); + let response = client(1) + .query(&addr, BootstrapRequest::new(&cluster, cap)) + .await + .unwrap(); + assert_eq!(response.endpoints.len(), cap); + assert!( + response + .endpoints + .contains(&format!("peer-{cap}.invalid:9000")) + ); + manager.shutdown().await.unwrap(); +} + +#[test] +fn manager_rejects_invalid_public_sync_config_before_channel_allocation() { + let store = temp_store(); + for (field, config) in invalid_sync_configs() { + let Err(error) = SyncManager::try_new_with_discovery( + NodeId::new("local-node"), + config, + Arc::clone(&store), + metrics(), + vec![DiscoveryProvider::new( + "untouched", + Arc::new(UntouchedDiscovery), + )], + DiscoveryRuntimeConfig::default(), + ) else { + panic!("invalid {field} was accepted"); + }; + assert!(matches!( + error.downcast_ref::(), + Some(crate::SyncConfigError::Invalid(_)) + )); + assert!(error.to_string().contains(field), "{error}"); + } +} + +#[tokio::test] +async fn manager_revalidates_timer_and_node_limits_before_touching_providers() { + for (field, config) in invalid_sync_configs() { + let mut manager = SyncManager::try_new_with_discovery( + NodeId::new("local-node"), + SyncConfig::new(), + temp_store(), + metrics(), + vec![DiscoveryProvider::new( + "untouched", + Arc::new(UntouchedDiscovery), + )], + DiscoveryRuntimeConfig::default(), + ) + .unwrap(); + manager.config = config.with_listen_addr("127.0.0.1:0"); + let error = manager.start().await.unwrap_err(); + assert!(error.to_string().contains(field), "{error}"); + assert!(manager.node.is_none()); + assert!(manager.discovery_coordinator.is_none()); + assert!(manager.op_rx.is_some()); + manager.shutdown().await.unwrap(); + } +} + +#[tokio::test] +async fn manager_rejects_invalid_bootstrap_policy_before_touching_providers() { + for discovery_config in [ + DiscoveryRuntimeConfig::default().with_cluster_id("x".repeat(256)), + DiscoveryRuntimeConfig::default().with_max_candidates(0), + ] { + let mut manager = SyncManager::try_new_with_discovery( + NodeId::new("local-node"), + SyncConfig::new().with_listen_addr("127.0.0.1:0"), + temp_store(), + metrics(), + vec![DiscoveryProvider::new( + "untouched", + Arc::new(UntouchedDiscovery), + )], + discovery_config, + ) + .unwrap(); + assert!(manager.start().await.is_err()); + assert!(manager.node.is_none()); + assert!(manager.discovery_coordinator.is_none()); + assert!(manager.op_rx.is_some()); + manager.shutdown().await.unwrap(); + } +} + #[tokio::test] async fn manager_hydrates_pncounter_registry_from_durable_state() { let store = temp_store(); @@ -1791,6 +2047,360 @@ async fn reconnect_loop_connects_configured_peer_that_starts_later() { assert_eq!(read_materialized(&store_b, key), 1); } +#[tokio::test] +async fn dynamic_candidate_connects_after_startup_with_an_empty_snapshot() { + let addr_a = free_addr(); + let addr_b = free_addr(); + let discovery = Arc::new(TestDynamicDiscovery::empty()); + let config_a = SyncConfig::new() + .with_listen_addr(addr_a) + .with_anti_entropy_interval(Duration::from_millis(10)) + .with_reconnect_backoff(Duration::from_millis(10), Duration::from_millis(50)); + let mut manager_a = SyncManager::try_new_with_discovery( + NodeId::generate(), + config_a, + temp_store(), + metrics(), + vec![DiscoveryProvider::new("test-dynamic", discovery.clone())], + DiscoveryRuntimeConfig::default(), + ) + .unwrap(); + manager_a.start().await.unwrap(); + assert_eq!(manager_a.connected_peer_count().await, 0); + + let config_b = SyncConfig::new().with_listen_addr(addr_b.clone()); + let (mut manager_b, handle_b, _store_b) = started_manager_with_config(config_b).await; + + discovery.add(addr_b.clone()); + wait_for_connected_peer(&manager_a).await; + + let handle_a = manager_a.handle(); + let deadline = Instant::now() + Duration::from_secs(5); + let connections = loop { + let connections = handle_a.active_connections().await; + if !connections.is_empty() { + break connections; + } + assert!( + Instant::now() < deadline, + "connection metadata was not published" + ); + sleep(Duration::from_millis(10)).await; + }; + assert_eq!(connections.len(), 1); + assert_eq!( + connections[0].dialed_endpoint.as_deref(), + Some(addr_b.as_str()) + ); + assert_eq!(connections[0].direction, ConnectionDirection::Outbound); + assert_eq!( + connections[0].identity.verification, + PeerIdentityVerification::Unverified + ); + + discovery.remove(&addr_b); + let deadline = Instant::now() + Duration::from_secs(5); + loop { + if manager_a.peer_candidates().is_empty() { + break; + } + assert!( + Instant::now() < deadline, + "removed discovery candidate remained visible" + ); + sleep(Duration::from_millis(10)).await; + } + assert_eq!( + manager_a.connected_peer_count().await, + 1, + "removing a candidate must not terminate an admitted connection" + ); + + // The connection remains an anti-entropy target even after its discovery + // contribution is removed. No broadcast can deliver this operation. + dropped_local_increment(&manager_b, &handle_b, "removed-candidate", 7).await; + wait_for_counter(&manager_a, "removed-candidate", 7).await; + assert_eq!(read_materialized(&manager_a.store, "removed-candidate"), 7); + + manager_a.shutdown().await.unwrap(); + manager_b.shutdown().await.unwrap(); +} + +#[tokio::test] +async fn anti_entropy_recovers_missing_ops_during_continuous_candidate_churn() { + let interval = Duration::from_millis(200); + // A controlled peer never broadcasts or answers connection-time requests. + let source_id = NodeId::generate(); + let mut source = Node::new(NodeConfig::new(source_id.clone(), "127.0.0.1:0")); + let mut source_events = source.take_event_receiver().unwrap(); + let source_addr = source.start_listener().await.unwrap().to_string(); + let discovery = Arc::new(TestDynamicDiscovery::empty()); + discovery.add(source_addr); + let mut target = SyncManager::try_new_with_discovery( + NodeId::generate(), + SyncConfig::new() + .with_listen_addr("127.0.0.1:0") + .with_max_peers(1) + .with_anti_entropy_interval(interval), + temp_store(), + metrics(), + vec![DiscoveryProvider::new("test-dynamic", discovery.clone())], + DiscoveryRuntimeConfig::default(), + ) + .unwrap(); + target.start().await.unwrap(); + wait_for_connected_peer(&target).await; + + // Reserve the unused endpoint; churn must not create additional connections. + let unused = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let unused_addr = unused.local_addr().unwrap().to_string(); + let mut candidates = target.discovery_coordinator.as_ref().unwrap().candidates(); + let (updates_tx, mut updates_rx) = tokio::sync::watch::channel(Instant::now()); + let churn = async { + let mut cadence = tokio::time::interval(Duration::from_millis(10)); + cadence.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut added = false; + loop { + cadence.tick().await; + added = !added; + if added { + discovery.add(unused_addr.clone()); + } else { + discovery.remove(&unused_addr); + } + // Observe each published change rather than just enqueueing events. + candidates + .wait_for(|snapshot| snapshot.contains(&unused_addr) == added) + .await + .unwrap(); + updates_tx.send_replace(Instant::now()); + } + }; + let recovery = async { + let mut ops = Vec::new(); + for expected in 1..=3 { + ops.push(Op::gcounter_increment(source_id.clone(), "churn", 1)); + updates_rx.changed().await.unwrap(); + let first_change = *updates_rx.borrow_and_update(); + // Observe distinct published changes spanning a full anti-entropy + // interval, rather than assuming a scheduler-dependent update count. + let last_change = *updates_rx + .wait_for(|changed_at| *changed_at >= first_change + interval) + .await + .unwrap(); + assert!(last_change.duration_since(first_change) >= interval); + + // Discard all earlier requests, including any eager startup pull. + // Only a fresh periodic request may recover this missing operation. + while let Ok(event) = source_events.try_recv() { + assert!(matches!( + event, + NodeEvent::PeerConnected { .. } | NodeEvent::PullRequested { .. } + )); + } + let (reply_addr, since) = wait_for_pull_request(&mut source_events).await; + assert_eq!(since, None); + let change_at_pull = *updates_rx.borrow_and_update(); + updates_rx.changed().await.unwrap(); + assert!(*updates_rx.borrow_and_update() > change_at_pull); + assert_eq!(target.get_counter_value("churn").await, expected - 1); + source + .send_ops_to_addr(&reply_addr, ops.clone()) + .await + .unwrap(); + + // FIFO on this connection makes the reply to our pull an apply + // barrier: the target must have processed the preceding ops first. + source + .send_pull_since_to_addr(&reply_addr, None) + .await + .unwrap(); + loop { + match source_events + .recv() + .await + .expect("peer event channel closed") + { + NodeEvent::OpsReceived { ops: received, .. } => { + assert_eq!(received, ops); + break; + } + NodeEvent::PullRequested { .. } => {} + event => panic!("unexpected event during recovery: {event:?}"), + } + } + assert_eq!(target.get_counter_value("churn").await, expected); + } + }; + tokio::time::timeout(Duration::from_secs(5), async { + tokio::select! { + _ = churn => unreachable!("churn must continue until recovery completes"), + _ = recovery => {} + } + }) + .await + .expect("anti-entropy did not recover missing ops while candidate updates continued"); + assert_eq!(target.connected_peer_count().await, 1); + assert_eq!(read_materialized(&target.store, "churn"), 3); + assert_eq!(target.op_log.read().await.len(), 3); + assert_eq!(target.seen_ops.read().await.len(), 3); + target.shutdown().await.unwrap(); + source.shutdown().await; +} + +#[tokio::test] +async fn anti_entropy_inbound_only_max_peers_one_recovers_older_missing_op() { + let addr = free_addr(); + let (mut target, _, store) = started_manager_with_config( + SyncConfig::new() + .with_listen_addr(addr.clone()) + .with_max_peers(1) + .with_anti_entropy_interval(Duration::from_millis(10)), + ) + .await; + let source_id = NodeId::generate(); + let mut source = Node::new(NodeConfig::new(source_id.clone(), "127.0.0.1:0")); + let mut source_events = source.take_event_receiver().unwrap(); + source.connect_to_peer(&addr).await.unwrap(); + wait_for_connected_peer(&target).await; + assert!(target.peer_candidates().is_empty()); + assert_eq!(target.connected_peer_count().await, 1); + + let older = Op::gcounter_increment(source_id.clone(), "missing", 3); + let newer = Op::gcounter_increment(source_id.clone(), "received", 7); + source + .send_ops_to_addr(&addr, vec![newer.clone()]) + .await + .unwrap(); + wait_for_counter(&target, "received", 7).await; + assert_eq!(target.get_counter_value("missing").await, 0); + assert_eq!( + target.anti_entropy_watermarks.read().await.get(&source_id), + Some(&newer.id.as_str().to_string()) + ); + let connections = target.handle().active_connections().await; + assert_eq!(connections.len(), 1); + assert_eq!(connections[0].direction, ConnectionDirection::Inbound); + assert!(connections[0].dialed_endpoint.is_none()); + + // A newer received op must not become a causal frontier. The inbound + // transport address (not an advertised candidate) is the only valid target. + let (reply_addr, since) = wait_for_pull_request(&mut source_events).await; + assert_eq!(since, None); + source + .send_ops_to_addr(&reply_addr, vec![older.clone(), newer.clone()]) + .await + .unwrap(); + wait_for_counter(&target, "missing", 3).await; + + // Repeated full bounded-log pulls must still deduplicate previously seen ops. + let (reply_addr, since) = wait_for_pull_request(&mut source_events).await; + assert_eq!(since, None); + let barrier = Op::gcounter_increment(source_id, "barrier", 1); + source + .send_ops_to_addr(&reply_addr, vec![older, newer, barrier]) + .await + .unwrap(); + wait_for_counter(&target, "barrier", 1).await; + assert_eq!(target.get_counter_value("missing").await, 3); + assert_eq!(target.get_counter_value("received").await, 7); + assert_eq!(read_materialized(&store, "missing"), 3); + assert_eq!(read_durable_gcounter_state(&store, "missing").value(), 3); + assert_eq!(target.op_log.read().await.len(), 3); + assert_eq!(target.seen_ops.read().await.len(), 3); + target.shutdown().await.unwrap(); + source.shutdown().await; +} + +#[tokio::test] +async fn initial_unresponsive_candidates_do_not_block_startup_or_shutdown() { + use tokio::io::AsyncReadExt; + + let first = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let second = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let first_addr = first.local_addr().unwrap().to_string(); + let second_addr = second.local_addr().unwrap().to_string(); + let mut manager = SyncManager::new( + NodeId::generate(), + SyncConfig::new() + .with_listen_addr("127.0.0.1:0") + .with_peer(first_addr.clone()) + .with_peer(second_addr.clone()) + .with_socket_timeout(Duration::from_secs(60)), + temp_store(), + metrics(), + ); + tokio::time::timeout(Duration::from_secs(1), manager.start()) + .await + .expect("startup waited for initial peer handshakes") + .unwrap(); + assert!(manager.start().await.is_err(), "start remains one-shot"); + assert!(manager.event_task.is_some()); + assert!(manager.broadcast_task.is_some()); + assert!(manager.reconnect_task.is_some()); + assert!(manager.anti_entropy_task.is_some()); + + let (mut stalled, _) = tokio::time::timeout(Duration::from_secs(1), first.accept()) + .await + .expect("initial reconnect dial did not start") + .unwrap(); + tokio::time::timeout(Duration::from_secs(1), stalled.read_exact(&mut [0; 1])) + .await + .expect("initial dial did not send its Hello") + .unwrap(); + let error = manager.connect_to_peer(&second_addr).await.unwrap_err(); + assert!(matches!( + error.downcast_ref::(), + Some(nx_net::NetError::ConnectionFailed(message)) + if message.contains("outbound connection attempt limit reached: 1") + )); + assert_eq!(manager.connected_peer_count().await, 0); + + // Broadcast persistence and event processing are already running while the + // first candidate has stalled and the second has not yet been attempted. + local_increment(&manager.handle(), "startup", 1).await; + tokio::time::timeout(Duration::from_secs(1), manager.shutdown()) + .await + .expect("shutdown waited for the in-flight dial timeout") + .unwrap(); + assert_eq!(manager.op_log.read().await.len(), 1); + assert!(manager.reconnect_task.is_none()); + assert!(manager.event_task.is_none()); + assert!(manager.broadcast_task.is_none()); + assert!(manager.anti_entropy_task.is_none()); + assert!( + manager.start().await.is_err(), + "shutdown must not allow restart" + ); + tokio::time::timeout(Duration::from_secs(1), stalled.read_to_end(&mut Vec::new())) + .await + .expect("canceled dial kept its transport open") + .unwrap(); + assert!( + tokio::time::timeout(Duration::from_millis(50), second.accept()) + .await + .is_err() + ); +} + +#[tokio::test] +async fn immediate_shutdown_cancels_initial_dial_before_task_first_poll() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let (mut manager, _, _) = started_manager_with_config( + SyncConfig::new() + .with_listen_addr("127.0.0.1:0") + .with_peer(listener.local_addr().unwrap().to_string()) + .with_socket_timeout(Duration::from_secs(60)), + ) + .await; + tokio::time::timeout(Duration::from_secs(1), manager.shutdown()) + .await + .expect("pre-signaled shutdown must not wait for an initial dial") + .unwrap(); + assert!(manager.reconnect_task.is_none()); + assert_eq!(manager.connected_peer_count().await, 0); +} + #[tokio::test] async fn anti_entropy_pull_converges_peer_that_missed_broadcast() { let key = "visits"; diff --git a/crates/nx-core/src/sync_manager/tests/support.rs b/crates/nx-core/src/sync_manager/tests/support.rs index e45d882..a2cb469 100644 --- a/crates/nx-core/src/sync_manager/tests/support.rs +++ b/crates/nx-core/src/sync_manager/tests/support.rs @@ -1,6 +1,66 @@ use super::*; use crate::sync_manager::schema::ensure_sync_schema; +/// Fails even on synchronous metadata/lifecycle access, not just on watch acquisition. +pub(super) struct UntouchedDiscovery; + +#[async_trait::async_trait] +impl crate::PeerDiscovery for UntouchedDiscovery { + fn cluster_id(&self) -> &str { + panic!("invalid local configuration must not inspect providers"); + } + + fn announcement_support(&self) -> crate::AnnouncementSupport { + panic!("invalid local configuration must not inspect providers"); + } + + async fn discover(&self) -> Result { + panic!("invalid local configuration must not start providers"); + } + + async fn watch(&self) -> Result { + panic!("invalid local configuration must not start providers"); + } + + async fn announce(&self, _: &crate::PeerAnnouncement) -> Result<(), crate::DiscoveryError> { + panic!("invalid local configuration must not announce"); + } + + fn request_shutdown(&self) { + panic!("unstarted providers must not need rollback"); + } +} + +pub(super) fn invalid_sync_configs() -> Vec<(&'static str, SyncConfig)> { + vec![ + ( + "reconnect_initial_delay", + SyncConfig::new().with_reconnect_backoff(Duration::MAX, Duration::from_secs(1)), + ), + ( + "reconnect_max_delay", + SyncConfig::new().with_reconnect_backoff(Duration::from_secs(1), Duration::MAX), + ), + ( + "anti_entropy_interval", + SyncConfig::new().with_anti_entropy_interval(Duration::MAX), + ), + ( + "socket_timeout", + SyncConfig::new().with_socket_timeout(Duration::MAX), + ), + ( + "socket_timeout", + SyncConfig::new().with_socket_timeout(Duration::ZERO), + ), + ( + "queued_ops_limit", + SyncConfig::new().with_queued_ops_limit(usize::MAX), + ), + ("max_peers", SyncConfig::new().with_max_peers(usize::MAX)), + ] +} + pub(super) fn temp_store() -> Arc { use std::sync::atomic::{AtomicU64, Ordering}; static COUNTER: AtomicU64 = AtomicU64::new(0); @@ -139,7 +199,7 @@ pub(super) fn test_event_context( "127.0.0.1:0", ))), peer_health, - peer_node_ids: Arc::new(RwLock::new(HashMap::new())), + active_connections: Arc::new(RwLock::new(HashMap::new())), anti_entropy_watermarks: Arc::new(RwLock::new(HashMap::new())), peer_dead_after_failures: 2, } @@ -462,6 +522,23 @@ pub(super) async fn wait_for_connected_peer(manager: &SyncManager) { } } +pub(super) async fn wait_for_pull_request( + events: &mut mpsc::Receiver, +) -> (String, Option) { + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if let NodeEvent::PullRequested { + addr, since_op_id, .. + } = events.recv().await.expect("peer event channel closed") + { + return (addr, since_op_id); + } + } + }) + .await + .expect("peer did not receive an anti-entropy pull") +} + pub(super) async fn wait_for_peer_health( manager: &SyncManager, peer: &str, diff --git a/crates/nx-core/src/sync_manager/types.rs b/crates/nx-core/src/sync_manager/types.rs index 30de259..b7ec541 100644 --- a/crates/nx-core/src/sync_manager/types.rs +++ b/crates/nx-core/src/sync_manager/types.rs @@ -2,7 +2,7 @@ use std::collections::{HashMap, HashSet, VecDeque}; use std::sync::{Arc, atomic::AtomicU64}; use std::time::Duration; -use nx_net::Node; +use nx_net::{Node, PeerConnectionInfo}; use nx_store::Store as NxStore; use nx_sync::{GCounter, LwwMap, LwwRegister, NodeId, ORSet, Op, PNCounter, Rga}; use tokio::sync::{RwLock, watch}; @@ -129,7 +129,7 @@ impl SeenOps { pub(super) struct ReconnectLoopContext { pub(super) node: Arc, - pub(super) peers: Vec, + pub(super) candidates_rx: watch::Receiver>>, pub(super) max_peers: usize, pub(super) initial_delay: Duration, pub(super) max_delay: Duration, @@ -141,7 +141,6 @@ pub(super) struct ReconnectLoopContext { pub(super) struct AntiEntropyLoopContext { pub(super) node: Arc, - pub(super) peers: Vec, pub(super) interval: Duration, pub(super) shutdown_rx: watch::Receiver, pub(super) metrics: Arc, @@ -226,7 +225,7 @@ pub(super) struct NodeEventContext { pub(super) metrics: Arc, pub(super) node: Arc, pub(super) peer_health: Arc>>, - pub(super) peer_node_ids: Arc>>, + pub(super) active_connections: Arc>>, pub(super) anti_entropy_watermarks: Arc>>, pub(super) peer_dead_after_failures: u32, } diff --git a/crates/nx-net/Cargo.toml b/crates/nx-net/Cargo.toml index db2cdcb..bec3920 100644 --- a/crates/nx-net/Cargo.toml +++ b/crates/nx-net/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nx-net" -version = "0.1.4" +version = "0.1.5" edition = "2024" license.workspace = true @@ -8,7 +8,7 @@ license.workspace = true fuzzing = [] [dependencies] -nx-sync = { version = "0.1.4", path = "../nx-sync" } +nx-sync = { version = "0.1.5", path = "../nx-sync" } serde = { version = "1", features = ["derive"] } serde_json = "1" wincode = { version = "0.6.0", features = ["derive"] } diff --git a/crates/nx-net/src/bootstrap.rs b/crates/nx-net/src/bootstrap.rs new file mode 100644 index 0000000..3f41d90 --- /dev/null +++ b/crates/nx-net/src/bootstrap.rs @@ -0,0 +1,1178 @@ +use std::collections::{HashMap, HashSet}; +use std::net::IpAddr; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use nx_sync::NodeId; +use tokio::sync::{OwnedSemaphorePermit, Semaphore}; + +use crate::message::{PROTOCOL_VERSION, ProtocolWireError, WireMessage, WireMessageKind}; +use crate::node::{ + connect_transport, read_wire_message_with_format, supported_formats_for, verify_peer_identity, + write_message, +}; +use crate::{BootstrapError, BootstrapResult, NetError, NetResult, SerializationFormat, TlsConfig}; + +/// Default maximum number of endpoint suggestions retained by a bootstrap seed. +pub const DEFAULT_BOOTSTRAP_CACHE_CAPACITY: usize = 1_024; +/// Default maximum number of endpoint suggestions returned by one bootstrap query. +pub const DEFAULT_BOOTSTRAP_RESPONSE_CAPACITY: usize = 128; +/// Hard upper bound for candidates requested or returned in one bootstrap query. +pub const MAX_BOOTSTRAP_RESPONSE_CAPACITY: usize = 4_096; +/// Default lifetime of an endpoint suggestion learned by a bootstrap seed. +pub const DEFAULT_BOOTSTRAP_CANDIDATE_TTL: Duration = Duration::from_secs(60); +/// Hard upper bound for a bootstrap candidate lease. +pub const MAX_BOOTSTRAP_CANDIDATE_TTL: Duration = Duration::from_secs(300); +/// Default maximum number of concurrent one-shot bootstrap queries per client. +pub const DEFAULT_MAX_CONCURRENT_BOOTSTRAP_QUERIES: usize = 1; + +pub(crate) const MAX_CLUSTER_ID_LEN: usize = 255; +pub(crate) const MAX_ENDPOINT_LEN: usize = 512; + +/// Server-side policy for the authenticated bootstrap exchange. +#[derive(Debug, Clone)] +pub struct BootstrapServerConfig { + cluster_id: String, + advertised_endpoint: Option, + max_cached_candidates: usize, + max_response_candidates: usize, + candidate_ttl: Duration, +} + +impl BootstrapServerConfig { + pub fn new(cluster_id: impl Into) -> BootstrapResult { + let config = Self { + cluster_id: cluster_id.into(), + advertised_endpoint: None, + max_cached_candidates: DEFAULT_BOOTSTRAP_CACHE_CAPACITY, + max_response_candidates: DEFAULT_BOOTSTRAP_RESPONSE_CAPACITY, + candidate_ttl: DEFAULT_BOOTSTRAP_CANDIDATE_TTL, + }; + config.validate()?; + Ok(config) + } + + pub fn with_advertised_endpoint( + mut self, + endpoint: impl Into, + ) -> BootstrapResult { + let endpoint = endpoint.into(); + self.advertised_endpoint = + Some(canonicalize_advertised_endpoint(&endpoint).map_err(bootstrap_config_error)?); + Ok(self) + } + + pub fn with_max_cached_candidates(mut self, limit: usize) -> BootstrapResult { + if limit == 0 { + return Err(BootstrapError::InvalidConfig( + "bootstrap cache capacity must be greater than zero".into(), + )); + } + self.max_cached_candidates = limit; + Ok(self) + } + + pub fn with_max_response_candidates(mut self, limit: usize) -> BootstrapResult { + validate_response_capacity(limit).map_err(bootstrap_config_error)?; + self.max_response_candidates = limit; + Ok(self) + } + + pub fn with_candidate_ttl(mut self, ttl: Duration) -> BootstrapResult { + validate_candidate_ttl(ttl).map_err(bootstrap_config_error)?; + self.candidate_ttl = ttl; + Ok(self) + } + + pub fn cluster_id(&self) -> &str { + &self.cluster_id + } + + pub fn advertised_endpoint(&self) -> Option<&str> { + self.advertised_endpoint.as_deref() + } + + pub fn max_cached_candidates(&self) -> usize { + self.max_cached_candidates + } + + pub fn max_response_candidates(&self) -> usize { + self.max_response_candidates + } + + pub fn candidate_ttl(&self) -> Duration { + self.candidate_ttl + } + + pub(crate) fn validate(&self) -> BootstrapResult<()> { + validate_cluster_id(&self.cluster_id).map_err(bootstrap_config_error)?; + if let Some(endpoint) = &self.advertised_endpoint { + validate_advertised_endpoint(endpoint).map_err(bootstrap_config_error)?; + } + if self.max_cached_candidates == 0 { + return Err(BootstrapError::InvalidConfig( + "bootstrap cache capacity must be greater than zero".into(), + )); + } + validate_response_capacity(self.max_response_candidates).map_err(bootstrap_config_error)?; + validate_candidate_ttl(self.candidate_ttl).map_err(bootstrap_config_error) + } +} + +/// Client-side identity, transport, and bounds for one-shot bootstrap queries. +#[derive(Debug, Clone)] +pub struct BootstrapClientConfig { + pub node_id: NodeId, + pub tls: Option, + pub max_message_size: usize, + pub socket_timeout: Duration, + pub serialization_format: SerializationFormat, + pub max_response_candidates: usize, + pub max_candidate_ttl: Duration, + pub max_concurrent_queries: usize, +} + +impl BootstrapClientConfig { + pub fn new(node_id: NodeId) -> Self { + Self { + node_id, + tls: None, + max_message_size: crate::DEFAULT_MAX_MESSAGE_SIZE, + socket_timeout: crate::DEFAULT_SOCKET_TIMEOUT, + serialization_format: SerializationFormat::Bincode, + max_response_candidates: DEFAULT_BOOTSTRAP_RESPONSE_CAPACITY, + max_candidate_ttl: MAX_BOOTSTRAP_CANDIDATE_TTL, + max_concurrent_queries: DEFAULT_MAX_CONCURRENT_BOOTSTRAP_QUERIES, + } + } + + pub(crate) fn validate(&self) -> BootstrapResult<()> { + if self.max_message_size == 0 { + return Err(BootstrapError::InvalidConfig( + "bootstrap maximum message size must be greater than zero".into(), + )); + } + if self.socket_timeout.is_zero() { + return Err(BootstrapError::InvalidConfig( + "bootstrap socket timeout must be greater than zero".into(), + )); + } + if Instant::now().checked_add(self.socket_timeout).is_none() { + return Err(BootstrapError::InvalidConfig( + "bootstrap socket timeout exceeds the supported deadline range".into(), + )); + } + validate_response_capacity(self.max_response_candidates).map_err(bootstrap_config_error)?; + validate_candidate_ttl(self.max_candidate_ttl).map_err(bootstrap_config_error)?; + if !(1..=Semaphore::MAX_PERMITS).contains(&self.max_concurrent_queries) { + return Err(BootstrapError::InvalidConfig(format!( + "bootstrap concurrent query limit must be in 1..={}", + Semaphore::MAX_PERMITS + ))); + } + Ok(()) + } +} + +/// A bounded request for candidate endpoints in one cluster. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BootstrapRequest { + pub cluster_id: String, + pub advertised_endpoint: Option, + pub max_results: usize, +} + +impl BootstrapRequest { + pub fn new(cluster_id: impl Into, max_results: usize) -> Self { + Self { + cluster_id: cluster_id.into(), + advertised_endpoint: None, + max_results, + } + } + + pub fn with_advertised_endpoint(mut self, endpoint: impl Into) -> Self { + self.advertised_endpoint = Some(endpoint.into()); + self + } +} + +/// Result of an authenticated one-shot bootstrap exchange. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BootstrapResponse { + /// Identity authenticated by the transport policy (certificate-bound with secure TLS). + pub seed_node_id: NodeId, + /// Advertised endpoints. These remain untrusted connection candidates. + pub endpoints: Vec, + /// Maximum time for which the returned snapshot may be retained without refresh. + pub candidate_ttl: Duration, +} + +/// Clonable client for bounded one-shot bootstrap exchanges. +#[derive(Debug, Clone)] +pub struct BootstrapClient { + pub(crate) config: Arc, + query_slots: Arc, +} + +impl BootstrapClient { + pub fn new(config: BootstrapClientConfig) -> BootstrapResult { + config.validate()?; + let max_concurrent_queries = config.max_concurrent_queries; + Ok(Self { + config: Arc::new(config), + query_slots: Arc::new(Semaphore::new(max_concurrent_queries)), + }) + } + + pub(crate) fn acquire_query_slot(&self) -> BootstrapResult { + Arc::clone(&self.query_slots) + .try_acquire_owned() + .map_err(|_| BootstrapError::ConcurrencyLimitReached { + limit: self.config.max_concurrent_queries, + }) + } + + /// Contact one seed and return its authenticated, bounded endpoint suggestions. + /// + /// Only the seed identity is authenticated here. Every returned endpoint + /// remains a candidate and must pass the normal peer handshake independently. + pub async fn query( + &self, + seed: &str, + request: BootstrapRequest, + ) -> BootstrapResult { + validate_cluster_id(&request.cluster_id).map_err(bootstrap_config_error)?; + if request.max_results == 0 + || request.max_results > self.config.max_response_candidates + || u32::try_from(request.max_results).is_err() + { + return Err(BootstrapError::InvalidConfig(format!( + "bootstrap max_results must be in 1..={}", + self.config.max_response_candidates + ))); + } + if let Some(endpoint) = &request.advertised_endpoint { + validate_advertised_endpoint(endpoint).map_err(bootstrap_config_error)?; + } + + let _query_slot = self.acquire_query_slot()?; + let (stream, _transport_addr) = + connect_transport(seed, self.config.tls.as_ref(), self.config.socket_timeout).await?; + let peer_cert = stream.peer_cert_der(); + // A bootstrap request may disclose our advertised endpoint. Apply the + // certificate allowlist before sending it, then bind the server's + // claimed NodeId to the same certificate after the response. + if self + .config + .tls + .as_ref() + .is_some_and(|configuration| !configuration.insecure) + { + let certificate = peer_cert.as_ref().ok_or_else(|| { + NetError::TlsError("missing peer certificate in TLS session".into()) + })?; + let expected_seed_id = crate::tls::derive_protocol_node_id_from_cert(certificate)?; + verify_peer_identity( + &self.config.node_id, + &expected_seed_id, + Some(certificate), + self.config.tls.as_ref(), + )?; + } + let (mut reader, mut writer) = tokio::io::split(stream); + let supported_formats = supported_formats_for(self.config.serialization_format); + let hello = WireMessage::bootstrap_hello( + self.config.node_id.clone(), + supported_formats.clone(), + self.config.serialization_format, + request.cluster_id.clone(), + request.advertised_endpoint, + request.max_results as u32, + ); + write_message( + &mut writer, + &hello, + self.config.serialization_format, + self.config.socket_timeout, + ) + .await?; + + let (response_format, response) = read_wire_message_with_format( + &mut reader, + self.config.max_message_size, + self.config.socket_timeout, + ) + .await?; + let (seed_node_id, selected_format, candidates, candidate_ttl_ms) = match response.kind { + WireMessageKind::BootstrapAck { + node_id, + protocol_version, + selected_format, + cluster_id, + candidates, + candidate_ttl_ms, + } => { + if protocol_version != PROTOCOL_VERSION { + return Err(NetError::Wire(crate::WireError::protocol_mismatch( + protocol_version, + )) + .into()); + } + if cluster_id != request.cluster_id { + return Err(BootstrapError::InvalidResponse( + "bootstrap response cluster ID does not match the request".into(), + )); + } + (node_id, selected_format, candidates, candidate_ttl_ms) + } + WireMessageKind::Error { + error: ProtocolWireError::BootstrapRejected { reason }, + } => return Err(BootstrapError::Rejected { reason }), + WireMessageKind::Error { error } => { + let error = crate::WireError::try_from(error).map_err(BootstrapError::from)?; + return Err(NetError::Wire(error).into()); + } + _ => { + return Err(BootstrapError::InvalidResponse( + "expected BootstrapAck from bootstrap seed".into(), + )); + } + }; + + if !supported_formats.contains(&selected_format) { + return Err(BootstrapError::InvalidResponse(format!( + "bootstrap seed selected unsupported serialization format: {selected_format:?}" + ))); + } + if response_format != selected_format { + return Err(BootstrapError::InvalidResponse( + "bootstrap ACK frame format does not match the selected serialization format" + .into(), + )); + } + verify_peer_identity( + &self.config.node_id, + &seed_node_id, + peer_cert.as_ref(), + self.config.tls.as_ref(), + )?; + if candidates.len() > request.max_results + || candidates.len() > self.config.max_response_candidates + { + return Err(BootstrapError::InvalidResponse( + "bootstrap response exceeds the negotiated candidate limit".into(), + )); + } + let candidate_ttl = Duration::from_millis(candidate_ttl_ms); + if candidate_ttl.is_zero() || candidate_ttl > self.config.max_candidate_ttl { + return Err(BootstrapError::InvalidResponse( + "bootstrap response contains an invalid candidate TTL".into(), + )); + } + let mut seen = HashSet::new(); + let mut normalized_candidates = Vec::with_capacity(candidates.len()); + for endpoint in candidates { + let endpoint = canonicalize_advertised_endpoint(&endpoint) + .map_err(|error| BootstrapError::InvalidResponse(error.to_string()))?; + if !seen.insert(endpoint.clone()) { + return Err(BootstrapError::InvalidResponse( + "bootstrap response contains duplicate endpoints".into(), + )); + } + normalized_candidates.push(endpoint); + } + + Ok(BootstrapResponse { + seed_node_id, + endpoints: normalized_candidates, + candidate_ttl, + }) + } +} + +fn bootstrap_config_error(error: NetError) -> BootstrapError { + BootstrapError::InvalidConfig(error.to_string()) +} + +#[derive(Debug, Clone)] +struct CachedCandidate { + endpoint: String, + expires_at: Instant, +} + +#[derive(Debug)] +struct BootstrapCache { + by_node: HashMap, + order: Vec, +} + +#[derive(Debug)] +pub(crate) struct BootstrapServer { + config: BootstrapServerConfig, + advertised_endpoint: Mutex>, + cache: Mutex, +} + +impl BootstrapServer { + pub(crate) fn new(config: BootstrapServerConfig) -> Self { + Self { + advertised_endpoint: Mutex::new(config.advertised_endpoint.clone()), + config, + cache: Mutex::new(BootstrapCache { + by_node: HashMap::new(), + order: Vec::new(), + }), + } + } + + pub(crate) fn cluster_id(&self) -> &str { + self.config.cluster_id() + } + + pub(crate) fn candidate_ttl(&self) -> Duration { + self.config.candidate_ttl() + } + + pub(crate) fn announce(&self, endpoint: String) -> NetResult<()> { + let endpoint = canonicalize_advertised_endpoint(&endpoint)?; + *self.advertised_endpoint.lock().map_err(cache_poisoned)? = Some(endpoint); + Ok(()) + } + + pub(crate) fn withdraw(&self) -> NetResult<()> { + *self.advertised_endpoint.lock().map_err(cache_poisoned)? = None; + Ok(()) + } + + pub(crate) fn clear(&self) { + if let Ok(mut advertised) = self.advertised_endpoint.lock() { + *advertised = None; + } + if let Ok(mut cache) = self.cache.lock() { + cache.by_node.clear(); + cache.order.clear(); + } + } + + pub(crate) fn exchange( + &self, + requester: &NodeId, + requester_endpoint: Option, + requested_results: usize, + ) -> NetResult> { + let now = Instant::now(); + let mut cache = self.cache.lock().map_err(cache_poisoned)?; + prune_expired(&mut cache, now); + + match requester_endpoint { + Some(endpoint) => { + let endpoint = canonicalize_advertised_endpoint(&endpoint)?; + let expires_at = now.checked_add(self.config.candidate_ttl).ok_or_else(|| { + NetError::InvalidMessage( + "bootstrap candidate TTL exceeds the supported deadline range".into(), + ) + })?; + if let Some(entry) = cache.by_node.get_mut(requester) { + entry.endpoint = endpoint; + entry.expires_at = expires_at; + } else if cache.by_node.len() < self.config.max_cached_candidates { + cache.order.push(requester.clone()); + cache.by_node.insert( + requester.clone(), + CachedCandidate { + endpoint, + expires_at, + }, + ); + } + } + None => { + cache.by_node.remove(requester); + cache.order.retain(|node_id| node_id != requester); + } + } + + let limit = requested_results + .min(self.config.max_response_candidates) + .min(MAX_BOOTSTRAP_RESPONSE_CAPACITY); + let advertised = self + .advertised_endpoint + .lock() + .map_err(cache_poisoned)? + .clone(); + // Reserve for locally available entries, never for a remote request's capacity. + let available = cache.by_node.len() - usize::from(cache.by_node.contains_key(requester)) + + usize::from(advertised.is_some()); + let mut endpoints = Vec::with_capacity(limit.min(available)); + let mut seen = HashSet::new(); + if limit > 0 + && let Some(endpoint) = advertised + && seen.insert(endpoint.clone()) + { + endpoints.push(endpoint); + } + for node_id in &cache.order { + if endpoints.len() >= limit { + break; + } + if node_id == requester { + continue; + } + let Some(candidate) = cache.by_node.get(node_id) else { + continue; + }; + if seen.insert(candidate.endpoint.clone()) { + endpoints.push(candidate.endpoint.clone()); + } + } + endpoints.truncate(limit); + Ok(endpoints) + } +} + +fn prune_expired(cache: &mut BootstrapCache, now: Instant) { + cache + .by_node + .retain(|_, candidate| candidate.expires_at > now); + cache + .order + .retain(|node_id| cache.by_node.contains_key(node_id)); +} + +fn cache_poisoned(_: std::sync::PoisonError) -> NetError { + NetError::ConnectionFailed("bootstrap cache is poisoned".into()) +} + +pub(crate) fn validate_cluster_id(cluster_id: &str) -> NetResult<()> { + if cluster_id.is_empty() || cluster_id.len() > MAX_CLUSTER_ID_LEN { + return Err(NetError::InvalidMessage(format!( + "bootstrap cluster ID length must be in 1..={MAX_CLUSTER_ID_LEN}" + ))); + } + if cluster_id.chars().any(char::is_control) { + return Err(NetError::InvalidMessage( + "bootstrap cluster ID must not contain control characters".into(), + )); + } + Ok(()) +} + +pub(crate) fn validate_advertised_endpoint(endpoint: &str) -> NetResult<()> { + canonicalize_advertised_endpoint(endpoint).map(|_| ()) +} + +fn canonicalize_advertised_endpoint(endpoint: &str) -> NetResult { + if endpoint.is_empty() || endpoint.len() > MAX_ENDPOINT_LEN || endpoint.trim() != endpoint { + return Err(NetError::InvalidMessage(format!( + "advertised endpoint must be non-empty, trimmed, and at most {MAX_ENDPOINT_LEN} bytes" + ))); + } + + let (host, port) = split_endpoint(endpoint)?; + if port == 0 { + return Err(NetError::InvalidMessage( + "advertised endpoint must not use port zero".into(), + )); + } + let canonical_host = if let Ok(ip) = host.parse::() { + let undialable = ip.is_unspecified() + || ip.is_multicast() + || matches!(ip, IpAddr::V4(address) if address.is_broadcast()) + || matches!(ip, IpAddr::V6(address) if address.is_unicast_link_local()); + if undialable { + return Err(NetError::InvalidMessage( + "advertised endpoint must use a dialable unicast IP address".into(), + )); + } + ip.to_string() + } else if !valid_dns_name(host) { + return Err(NetError::InvalidMessage( + "advertised endpoint host must be a valid DNS name or IP address".into(), + )); + } else { + host.strip_suffix('.').unwrap_or(host).to_ascii_lowercase() + }; + if canonical_host.contains(':') { + Ok(format!("[{canonical_host}]:{port}")) + } else { + Ok(format!("{canonical_host}:{port}")) + } +} + +fn split_endpoint(endpoint: &str) -> NetResult<(&str, u16)> { + let (host, port) = if endpoint.starts_with('[') { + let closing = endpoint.find(']').ok_or_else(|| { + NetError::InvalidMessage("invalid bracketed advertised endpoint".into()) + })?; + if endpoint.as_bytes().get(closing + 1) != Some(&b':') { + return Err(NetError::InvalidMessage( + "advertised endpoint must include a port".into(), + )); + } + (&endpoint[1..closing], &endpoint[closing + 2..]) + } else { + let (host, port) = endpoint.rsplit_once(':').ok_or_else(|| { + NetError::InvalidMessage("advertised endpoint must be host:port".into()) + })?; + if host.contains(':') { + return Err(NetError::InvalidMessage( + "IPv6 advertised endpoints must use brackets".into(), + )); + } + (host, port) + }; + let port = port.parse::().map_err(|_| { + NetError::InvalidMessage("advertised endpoint port must be a valid u16".into()) + })?; + Ok((host, port)) +} + +fn valid_dns_name(host: &str) -> bool { + let host = host.strip_suffix('.').unwrap_or(host); + !host.is_empty() + && host.len() <= 253 + && host.split('.').all(|label| { + !label.is_empty() + && label.len() <= 63 + && label + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || byte == b'-') + && label + .as_bytes() + .first() + .is_some_and(u8::is_ascii_alphanumeric) + && label + .as_bytes() + .last() + .is_some_and(u8::is_ascii_alphanumeric) + }) +} + +fn validate_response_capacity(limit: usize) -> NetResult<()> { + if !(1..=MAX_BOOTSTRAP_RESPONSE_CAPACITY).contains(&limit) { + return Err(NetError::InvalidMessage(format!( + "bootstrap response capacity must be in 1..={MAX_BOOTSTRAP_RESPONSE_CAPACITY}" + ))); + } + Ok(()) +} + +fn validate_candidate_ttl(ttl: Duration) -> NetResult<()> { + if ttl.is_zero() || ttl > MAX_BOOTSTRAP_CANDIDATE_TTL { + return Err(NetError::InvalidMessage(format!( + "bootstrap candidate TTL must be in 1ms..={}s", + MAX_BOOTSTRAP_CANDIDATE_TTL.as_secs() + ))); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::node::read_wire_message_with_format; + use crate::{Node, NodeConfig, TestPki}; + + fn certificate_node_id(path: &std::path::Path) -> NodeId { + let certificate = crate::TlsConfig::load_certs(path).unwrap().remove(0); + crate::tls::derive_protocol_node_id_from_cert(&certificate).unwrap() + } + + #[test] + fn concurrent_query_capacity_is_validated_before_semaphore_construction() { + for limit in [0, Semaphore::MAX_PERMITS + 1, usize::MAX] { + let mut config = BootstrapClientConfig::new(NodeId::new("client")); + config.max_concurrent_queries = limit; + assert!(matches!( + config.validate(), + Err(BootstrapError::InvalidConfig(_)) + )); + assert!(matches!( + BootstrapClient::new(config), + Err(BootstrapError::InvalidConfig(_)) + )); + } + // A semaphore stores a permit count, not an allocation per permit. + for limit in [1, Semaphore::MAX_PERMITS - 1, Semaphore::MAX_PERMITS] { + let mut config = BootstrapClientConfig::new(NodeId::new("client")); + config.max_concurrent_queries = limit; + config.validate().unwrap(); + let client = BootstrapClient::new(config).unwrap(); + let permit = client.acquire_query_slot().unwrap(); + assert_eq!(client.query_slots.available_permits(), limit - 1); + drop(permit); + assert_eq!(client.query_slots.available_permits(), limit); + } + } + + #[test] + fn socket_timeout_must_have_a_representable_nonzero_deadline() { + for socket_timeout in [Duration::ZERO, Duration::MAX] { + let mut config = BootstrapClientConfig::new(NodeId::new("client")); + config.socket_timeout = socket_timeout; + assert!(matches!( + config.validate(), + Err(BootstrapError::InvalidConfig(_)) + )); + assert!(matches!( + BootstrapClient::new(config), + Err(BootstrapError::InvalidConfig(_)) + )); + } + for socket_timeout in [Duration::from_nanos(1), crate::DEFAULT_SOCKET_TIMEOUT] { + let mut config = BootstrapClientConfig::new(NodeId::new("client")); + config.socket_timeout = socket_timeout; + BootstrapClient::new(config).unwrap(); + } + } + + #[test] + fn overflowing_candidate_deadline_rejects_insertion_and_preserves_existing_lease() { + let mut server = BootstrapServer::new(BootstrapServerConfig::new("cluster-a").unwrap()); + let requester = NodeId::new("client"); + server + .exchange(&requester, Some("old.example:9000".into()), 1) + .unwrap(); + let original_expiry = server.cache.lock().unwrap().by_node[&requester].expires_at; + // Bypass public validation to exercise the defensive deadline calculation. + server.config.candidate_ttl = Duration::MAX; + for node_id in [&requester, &NodeId::new("new-client")] { + assert!(matches!( + server.exchange(node_id, Some("new.example:9000".into()), 1), + Err(NetError::InvalidMessage(_)) + )); + } + let cache = server.cache.lock().unwrap(); + assert_eq!(cache.by_node.len(), 1); + assert_eq!(cache.order.as_slice(), std::slice::from_ref(&requester)); + assert_eq!(cache.by_node[&requester].endpoint, "old.example:9000"); + assert_eq!(cache.by_node[&requester].expires_at, original_expiry); + } + + #[test] + fn response_capacity_is_validated_by_both_configurations_and_client_constructor() { + for limit in [ + 0, + MAX_BOOTSTRAP_RESPONSE_CAPACITY + 1, + u32::MAX as usize, + usize::MAX, + ] { + assert!( + BootstrapServerConfig::new("cluster-a") + .unwrap() + .with_max_response_candidates(limit) + .is_err() + ); + let mut server_config = BootstrapServerConfig::new("cluster-a").unwrap(); + server_config.max_response_candidates = limit; + assert!(server_config.validate().is_err()); + let mut client_config = BootstrapClientConfig::new(NodeId::new("client")); + client_config.max_response_candidates = limit; + assert!(client_config.validate().is_err()); + assert!(BootstrapClient::new(client_config).is_err()); + } + for limit in [1, MAX_BOOTSTRAP_RESPONSE_CAPACITY] { + BootstrapServerConfig::new("cluster-a") + .unwrap() + .with_max_response_candidates(limit) + .unwrap() + .validate() + .unwrap(); + let mut config = BootstrapClientConfig::new(NodeId::new("client")); + config.max_response_candidates = limit; + BootstrapClient::new(config).unwrap(); + } + } + + #[test] + fn huge_request_reserves_only_available_candidates() { + let server = BootstrapServer::new( + BootstrapServerConfig::new("cluster-a") + .unwrap() + .with_max_cached_candidates(usize::MAX) + .unwrap() + .with_max_response_candidates(MAX_BOOTSTRAP_RESPONSE_CAPACITY) + .unwrap(), + ); + let requester = NodeId::new("client"); + let empty = server + .exchange(&requester, None, u32::MAX as usize) + .unwrap(); + assert_eq!(empty.capacity(), 0); + server.announce("seed.example:9000".into()).unwrap(); + server + .exchange(&NodeId::new("peer"), Some("peer.example:9000".into()), 1) + .unwrap(); + let response = server + .exchange( + &requester, + Some("client.example:9000".into()), + u32::MAX as usize, + ) + .unwrap(); + assert_eq!(response, ["seed.example:9000", "peer.example:9000"]); + assert_eq!(response.capacity(), 2); + let zero = server.exchange(&requester, None, 0).unwrap(); + assert_eq!(zero.capacity(), 0); + assert!(zero.is_empty()); + } + + #[tokio::test] + async fn remote_u32_max_request_returns_a_bounded_v5_response() { + for format in [SerializationFormat::Bincode, SerializationFormat::Json] { + let node = Node::try_new_with_bootstrap_server( + NodeConfig::new(NodeId::new("seed"), "127.0.0.1:0"), + BootstrapServerConfig::new("cluster-a") + .unwrap() + .with_max_response_candidates(MAX_BOOTSTRAP_RESPONSE_CAPACITY) + .unwrap(), + ) + .unwrap(); + let bound = node.start_listener().await.unwrap(); + node.announce_bootstrap_endpoint(bound.to_string()).unwrap(); + let mut stream = tokio::net::TcpStream::connect(bound).await.unwrap(); + let request = WireMessage::bootstrap_hello( + NodeId::new("client"), + vec![format], + format, + "cluster-a".into(), + None, + u32::MAX, + ); + write_message(&mut stream, &request, format, Duration::from_secs(1)) + .await + .unwrap(); + let (_, response) = + read_wire_message_with_format(&mut stream, 1024, Duration::from_secs(1)) + .await + .unwrap(); + match response.kind { + WireMessageKind::BootstrapAck { + protocol_version, + candidates, + .. + } => { + assert_eq!(protocol_version, 5); + assert_eq!(candidates, [bound.to_string()]); + } + other => panic!("unexpected bootstrap reply: {other:?}"), + } + assert_eq!(node.connected_peer_count().await, 0); + node.shutdown().await; + } + } + + #[test] + fn endpoint_validation_rejects_wildcard_and_dynamic_port() { + assert!(validate_advertised_endpoint("0.0.0.0:9000").is_err()); + assert!(validate_advertised_endpoint("[::]:9000").is_err()); + assert!(validate_advertised_endpoint("[::0]:9000").is_err()); + assert!(validate_advertised_endpoint("[0:0:0:0:0:0:0:0]:9000").is_err()); + assert!(validate_advertised_endpoint("[fe80::1]:9000").is_err()); + assert!(validate_advertised_endpoint("*:9000").is_err()); + assert!(validate_advertised_endpoint("bad_name:9000").is_err()); + assert!(validate_advertised_endpoint("::1:9000").is_err()); + assert!(validate_advertised_endpoint("localhost:0").is_err()); + assert!(validate_advertised_endpoint("node.example:9000").is_ok()); + assert!(validate_advertised_endpoint("[2001:db8::1]:9000").is_ok()); + } + + #[test] + fn cache_is_bounded_deduplicated_and_excludes_requester() { + let server = BootstrapServer::new( + BootstrapServerConfig::new("cluster-a") + .unwrap() + .with_advertised_endpoint("seed.example:9000") + .unwrap() + .with_max_cached_candidates(2) + .unwrap(), + ); + + server + .exchange(&NodeId::new("node-a"), Some("a.example:9000".into()), 10) + .unwrap(); + server + .exchange(&NodeId::new("node-b"), Some("b.example:9000".into()), 10) + .unwrap(); + server + .exchange(&NodeId::new("node-c"), Some("c.example:9000".into()), 10) + .unwrap(); + + let response = server.exchange(&NodeId::new("node-a"), None, 2).unwrap(); + assert_eq!(response, vec!["seed.example:9000", "b.example:9000"]); + } + + #[tokio::test] + async fn one_shot_query_returns_candidates_without_registering_a_peer() { + let server_config = BootstrapServerConfig::new("cluster-a") + .unwrap() + .with_max_response_candidates(4) + .unwrap(); + let mut node = Node::try_new_with_bootstrap_server( + NodeConfig::new(NodeId::new("seed"), "127.0.0.1:0"), + server_config, + ) + .unwrap(); + let mut events = node.take_event_receiver().unwrap(); + let bound = node.start_listener().await.unwrap(); + node.announce_bootstrap_endpoint(bound.to_string()).unwrap(); + + let client = + BootstrapClient::new(BootstrapClientConfig::new(NodeId::new("client"))).unwrap(); + let response = client + .query( + &bound.to_string(), + BootstrapRequest::new("cluster-a", 4).with_advertised_endpoint("127.0.0.1:43111"), + ) + .await + .unwrap(); + + assert_eq!(response.seed_node_id, NodeId::new("seed")); + assert_eq!(response.endpoints, [bound.to_string()]); + assert_eq!(node.connected_peer_count().await, 0); + assert!( + tokio::time::timeout(Duration::from_millis(50), events.recv()) + .await + .is_err() + ); + node.shutdown().await; + } + + #[tokio::test] + async fn one_shot_query_supports_json_negotiation() { + let server_config = BootstrapServerConfig::new("cluster-a").unwrap(); + let node = Node::try_new_with_bootstrap_server( + NodeConfig::new(NodeId::new("seed"), "127.0.0.1:0") + .with_serialization_format(SerializationFormat::Json), + server_config, + ) + .unwrap(); + let bound = node.start_listener().await.unwrap(); + node.announce_bootstrap_endpoint(bound.to_string()).unwrap(); + let mut client_config = BootstrapClientConfig::new(NodeId::new("client")); + client_config.serialization_format = SerializationFormat::Json; + let client = BootstrapClient::new(client_config).unwrap(); + + let response = client + .query(&bound.to_string(), BootstrapRequest::new("cluster-a", 1)) + .await + .unwrap(); + + assert_eq!(response.endpoints, [bound.to_string()]); + assert_eq!(node.connected_peer_count().await, 0); + node.shutdown().await; + } + + #[tokio::test] + async fn cluster_mismatch_is_rejected_without_populating_the_cache() { + let server_config = BootstrapServerConfig::new("cluster-a").unwrap(); + let node = Node::try_new_with_bootstrap_server( + NodeConfig::new(NodeId::new("seed"), "127.0.0.1:0"), + server_config, + ) + .unwrap(); + let bound = node.start_listener().await.unwrap(); + let client = + BootstrapClient::new(BootstrapClientConfig::new(NodeId::new("client"))).unwrap(); + + let error = client + .query( + &bound.to_string(), + BootstrapRequest::new("cluster-b", 1).with_advertised_endpoint("127.0.0.1:43111"), + ) + .await + .unwrap_err(); + + assert!(matches!(error, BootstrapError::Rejected { .. })); + assert_eq!(node.connected_peer_count().await, 0); + node.shutdown().await; + } + + async fn query_ack_with_formats( + selected_format: SerializationFormat, + frame_format: SerializationFormat, + ) -> BootstrapResult { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let bound = listener.local_addr().unwrap(); + let seed = tokio::spawn(async move { + let (mut stream, _) = tokio::time::timeout(Duration::from_secs(2), listener.accept()) + .await + .unwrap() + .unwrap(); + let (_, hello) = + read_wire_message_with_format(&mut stream, 4096, Duration::from_secs(1)) + .await + .unwrap(); + assert!(matches!( + hello.kind, + WireMessageKind::BootstrapHello { + protocol_version: 5, + .. + } + )); + let ack = WireMessage::bootstrap_ack( + NodeId::new("seed"), + selected_format, + "cluster-a".into(), + vec!["seed.example:9000".into()], + 1_000, + ); + write_message(&mut stream, &ack, frame_format, Duration::from_secs(1)) + .await + .unwrap(); + }); + let mut config = BootstrapClientConfig::new(NodeId::new("client")); + config.socket_timeout = Duration::from_secs(1); + let result = BootstrapClient::new(config) + .unwrap() + .query(&bound.to_string(), BootstrapRequest::new("cluster-a", 1)) + .await; + seed.await.unwrap(); + result + } + + #[tokio::test] + async fn bootstrap_ack_rejects_mismatched_frame_format_in_both_encodings() { + for (selected, frame) in [ + (SerializationFormat::Json, SerializationFormat::Bincode), + (SerializationFormat::Bincode, SerializationFormat::Json), + ] { + let error = query_ack_with_formats(selected, frame).await.unwrap_err(); + assert!(matches!( + error, + BootstrapError::InvalidResponse(reason) if reason.contains("ACK frame format") + )); + } + } + + #[tokio::test] + async fn bootstrap_ack_accepts_matching_frame_format_in_both_encodings() { + for format in [SerializationFormat::Json, SerializationFormat::Bincode] { + let response = query_ack_with_formats(format, format).await.unwrap(); + assert_eq!(response.seed_node_id, NodeId::new("seed")); + assert_eq!(response.endpoints, ["seed.example:9000"]); + assert_eq!(response.candidate_ttl, Duration::from_secs(1)); + } + } + + #[tokio::test] + async fn bootstrap_client_allowlist_rejects_seed_before_announcement_disclosure() { + let pki = TestPki::generate().unwrap(); + let seed_id = certificate_node_id(&pki.dir_path().join("node1.pem")); + let client_id = certificate_node_id(&pki.dir_path().join("node2.pem")); + let seed = Node::try_new_with_bootstrap_server( + NodeConfig::new(seed_id.clone(), "127.0.0.1:0").with_tls(pki.node1_config()), + BootstrapServerConfig::new("cluster-a").unwrap(), + ) + .unwrap(); + let bound = seed.start_listener().await.unwrap(); + seed.announce_bootstrap_endpoint(bound.to_string()).unwrap(); + let seed_endpoint = format!("localhost:{}", bound.port()); + + let denied_tls = pki + .node2_config() + .with_allowed_peers(HashSet::from(["00000000000000000000000000000000".into()])); + let mut denied_config = BootstrapClientConfig::new(client_id.clone()); + denied_config.tls = Some(denied_tls); + let denied = BootstrapClient::new(denied_config).unwrap(); + let error = denied + .query( + &seed_endpoint, + BootstrapRequest::new("cluster-a", 4).with_advertised_endpoint("127.0.0.1:43111"), + ) + .await + .unwrap_err(); + assert!(matches!( + error, + BootstrapError::Transport(NetError::TlsError(_)) + )); + + let mut allowed_config = BootstrapClientConfig::new(client_id); + allowed_config.tls = Some(pki.node2_config()); + let allowed = BootstrapClient::new(allowed_config).unwrap(); + let response = allowed + .query(&seed_endpoint, BootstrapRequest::new("cluster-a", 4)) + .await + .unwrap(); + assert_eq!(response.seed_node_id, seed_id); + assert_eq!(response.endpoints, [bound.to_string()]); + seed.shutdown().await; + } + + #[tokio::test] + async fn bootstrap_seed_allowlist_rejects_ca_trusted_requester_without_caching_endpoint() { + let pki = TestPki::generate().unwrap(); + let seed_id = certificate_node_id(&pki.dir_path().join("node1.pem")); + let denied_id = certificate_node_id(&pki.dir_path().join("node2.pem")); + let (allowed_cert, allowed_key) = + crate::tls::generate_signed(&pki.ca_cert, &pki.ca_key, "allowed-client").unwrap(); + let cert_path = pki.dir_path().join("allowed.pem"); + let key_path = pki.dir_path().join("allowed-key.pem"); + crate::tls::write_cert_files(&allowed_cert, &allowed_key, &cert_path, &key_path).unwrap(); + let allowed_id = certificate_node_id(&cert_path); + assert_ne!(allowed_id, denied_id); + let seed_tls = pki + .node1_config() + .with_allowed_peers(HashSet::from([allowed_id.to_string()])); + assert!(!seed_tls.is_peer_allowed(&denied_id.to_string())); + let seed = Node::try_new_with_bootstrap_server( + NodeConfig::new(seed_id.clone(), "127.0.0.1:0").with_tls(seed_tls), + BootstrapServerConfig::new("cluster-a").unwrap(), + ) + .unwrap(); + let bound = seed.start_listener().await.unwrap(); + seed.announce_bootstrap_endpoint(bound.to_string()).unwrap(); + let seed_endpoint = format!("localhost:{}", bound.port()); + let socket_timeout = Duration::from_secs(10); + + // Complete CA-verified TLS first: rejection must be at the node allowlist, + // not at certificate validation or a client-side seed allowlist. + let (mut denied_stream, _) = + connect_transport(&seed_endpoint, Some(&pki.node2_config()), socket_timeout) + .await + .unwrap(); + let denied_hello = WireMessage::bootstrap_hello( + denied_id, + vec![SerializationFormat::Bincode], + SerializationFormat::Bincode, + "cluster-a".into(), + Some("denied.example:43111".into()), + 4, + ); + write_message( + &mut denied_stream, + &denied_hello, + SerializationFormat::Bincode, + socket_timeout, + ) + .await + .unwrap(); + let denied_result = + read_wire_message_with_format(&mut denied_stream, 4096, socket_timeout).await; + assert!( + denied_result.is_err(), + "allowlisted bootstrap seed returned a response to a denied requester" + ); + drop(denied_stream); + + // Use a different authenticated identity: a query without an announcement + // must not withdraw (and thereby hide) a cached entry for the denied node. + let mut allowed_config = BootstrapClientConfig::new(allowed_id); + allowed_config.tls = Some(TlsConfig::new( + cert_path.to_string_lossy(), + key_path.to_string_lossy(), + pki.dir_path().join("ca.pem").to_string_lossy(), + )); + allowed_config.socket_timeout = socket_timeout; + let response = BootstrapClient::new(allowed_config) + .unwrap() + .query(&seed_endpoint, BootstrapRequest::new("cluster-a", 4)) + .await + .unwrap(); + assert_eq!(response.seed_node_id, seed_id); + assert_eq!(response.endpoints, [bound.to_string()]); + assert_eq!(seed.connected_peer_count().await, 0); + seed.shutdown().await; + } +} diff --git a/crates/nx-net/src/error.rs b/crates/nx-net/src/error.rs index c6da2fb..36143c4 100644 --- a/crates/nx-net/src/error.rs +++ b/crates/nx-net/src/error.rs @@ -3,6 +3,7 @@ use thiserror::Error; use crate::message::WireError; pub type NetResult = Result; +pub type BootstrapResult = Result; #[derive(Debug, Error)] pub enum NetError { @@ -51,3 +52,44 @@ pub enum NetError { #[error("node ID mismatch: expected {expected}, got {got}")] NodeIdMismatch { expected: String, got: String }, } + +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum NodeConfigError { + #[error("max_peers must not exceed {limit}")] + MaxPeersTooLarge { limit: usize }, + + #[error("event_channel_capacity must be in 1..={limit}")] + InvalidEventChannelCapacity { limit: usize }, + + #[error("socket_timeout must be positive and form a representable deadline")] + InvalidSocketTimeout, +} + +#[derive(Debug, Error)] +#[non_exhaustive] +pub enum BootstrapError { + #[error("invalid bootstrap configuration: {0}")] + InvalidConfig(String), + + #[error("bootstrap query concurrency limit reached: {limit}")] + ConcurrencyLimitReached { limit: usize }, + + #[error("bootstrap request rejected: {reason}")] + Rejected { reason: String }, + + #[error("invalid bootstrap response: {0}")] + InvalidResponse(String), + + #[error("bootstrap transport error: {0}")] + Transport(#[source] NetError), + + #[error("invalid node configuration: {0}")] + NodeConfig(#[from] NodeConfigError), +} + +impl From for BootstrapError { + fn from(error: NetError) -> Self { + Self::Transport(error) + } +} diff --git a/crates/nx-net/src/lib.rs b/crates/nx-net/src/lib.rs index 2eba570..85772a4 100644 --- a/crates/nx-net/src/lib.rs +++ b/crates/nx-net/src/lib.rs @@ -1,10 +1,17 @@ +mod bootstrap; mod error; mod message; mod node; mod peer; mod tls; -pub use error::{NetError, NetResult}; +pub use bootstrap::{ + BootstrapClient, BootstrapClientConfig, BootstrapRequest, BootstrapResponse, + BootstrapServerConfig, DEFAULT_BOOTSTRAP_CACHE_CAPACITY, DEFAULT_BOOTSTRAP_CANDIDATE_TTL, + DEFAULT_BOOTSTRAP_RESPONSE_CAPACITY, DEFAULT_MAX_CONCURRENT_BOOTSTRAP_QUERIES, + MAX_BOOTSTRAP_CANDIDATE_TTL, MAX_BOOTSTRAP_RESPONSE_CAPACITY, +}; +pub use error::{BootstrapError, BootstrapResult, NetError, NetResult, NodeConfigError}; pub use message::{ Message, MessageKind, PROTOCOL_VERSION, SerializationFormat, WireError, WireRetryPolicy, }; @@ -15,7 +22,10 @@ pub use node::{ DEFAULT_MAX_MESSAGE_SIZE, DEFAULT_MAX_PEERS, DEFAULT_SOCKET_TIMEOUT, Node, NodeConfig, NodeEvent, }; -pub use peer::{PeerId, PeerInfo}; +pub use peer::{ + ConnectionDirection, PeerConnectionInfo, PeerId, PeerIdentity, PeerIdentityVerification, + PeerInfo, +}; pub use tls::{ NetStream, NodeId, TestPki, TlsConfig, derive_node_id, generate_ca, generate_self_signed, generate_signed, node_id_from_hex, node_id_to_hex, write_cert_files, diff --git a/crates/nx-net/src/message.rs b/crates/nx-net/src/message.rs index 2efc12a..983406e 100644 --- a/crates/nx-net/src/message.rs +++ b/crates/nx-net/src/message.rs @@ -5,7 +5,7 @@ use nx_sync::{NodeId, Op}; use serde::{Deserialize, Serialize}; /// Protocol version. -pub const PROTOCOL_VERSION: u32 = 4; +pub const PROTOCOL_VERSION: u32 = 5; const FORMAT_JSON: u8 = 0x01; const FORMAT_BINCODE: u8 = 0x02; @@ -168,6 +168,88 @@ pub struct Message { pub kind: MessageKind, } +#[derive( + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, wincode::SchemaRead, wincode::SchemaWrite, +)] +pub(crate) enum ProtocolWireError { + ProtocolMismatch { expected: u32, got: u32 }, + OpRejected { reason: String }, + RateLimited { retry_after_ms: Option }, + NotAuthorized { reason: String }, + Internal { reason: String }, + BootstrapRejected { reason: String }, +} + +impl std::fmt::Display for ProtocolWireError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::BootstrapRejected { reason } => { + write!(formatter, "bootstrap request rejected: {reason}") + } + error => WireError::try_from(error.clone()) + .map_err(|_| std::fmt::Error)? + .fmt(formatter), + } + } +} + +#[derive( + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, wincode::SchemaRead, wincode::SchemaWrite, +)] +pub(crate) enum WireMessageKind { + Hello { + node_id: NodeId, + #[serde(alias = "version")] + protocol_version: u32, + supported_formats: Vec, + preferred_format: SerializationFormat, + }, + HelloAck { + node_id: NodeId, + #[serde(alias = "version")] + protocol_version: u32, + selected_format: SerializationFormat, + }, + PushOps { + ops: Vec, + }, + PushOpsAck { + received_count: u64, + }, + PullSince { + since_op_id: Option, + }, + Ping, + Pong, + Error { + error: ProtocolWireError, + }, + BootstrapHello { + node_id: NodeId, + protocol_version: u32, + supported_formats: Vec, + preferred_format: SerializationFormat, + cluster_id: String, + advertised_endpoint: Option, + max_results: u32, + }, + BootstrapAck { + node_id: NodeId, + protocol_version: u32, + selected_format: SerializationFormat, + cluster_id: String, + candidates: Vec, + candidate_ttl_ms: u64, + }, +} + +#[derive( + Debug, Clone, PartialEq, Eq, Serialize, Deserialize, wincode::SchemaRead, wincode::SchemaWrite, +)] +pub(crate) struct WireMessage { + pub(crate) kind: WireMessageKind, +} + impl Message { pub fn hello(node_id: NodeId) -> Self { Self::hello_with_formats( @@ -256,24 +338,16 @@ impl Message { /// Serialize to bytes (length-prefixed format byte + payload). pub fn to_bytes_with_format(&self, format: SerializationFormat) -> NetResult> { - let payload = match format { - SerializationFormat::Json => serde_json::to_vec(self)?, - SerializationFormat::Bincode => wincode::config::serialize( - self, - wincode::config::Configuration::default().disable_preallocation_size_limit(), - )?, - }; - let len = payload - .len() - .checked_add(1) - .and_then(|len| u32::try_from(len).ok()) - .ok_or_else(|| NetError::InvalidMessage("message payload exceeds u32".to_string()))?; - let len = len.to_be_bytes(); - let mut buf = Vec::with_capacity(4 + 1 + payload.len()); - buf.extend_from_slice(&len); - buf.push(format.to_wire_byte()); - buf.extend_from_slice(&payload); - Ok(buf) + encode_frame( + format, + || serde_json::to_vec(self).map_err(NetError::from), + || { + Ok(wincode::config::serialize( + self, + wincode::config::Configuration::default().disable_preallocation_size_limit(), + )?) + }, + ) } /// Deserialize from bytes without the length prefix. @@ -284,21 +358,240 @@ impl Message { /// Deserialize from bytes without the length prefix, returning the detected format. pub fn from_bytes_with_format(bytes: &[u8]) -> NetResult<(SerializationFormat, Self)> { - let Some((&format_byte, payload)) = bytes.split_first() else { - return Err(NetError::InvalidMessage( - "message payload is missing serialization format byte".to_string(), - )); + decode_frame( + bytes, + |payload| serde_json::from_slice(payload), + deserialize_binary_message, + ) + } +} + +impl WireMessage { + pub(crate) fn bootstrap_hello( + node_id: NodeId, + supported_formats: Vec, + preferred_format: SerializationFormat, + cluster_id: String, + advertised_endpoint: Option, + max_results: u32, + ) -> Self { + Self { + kind: WireMessageKind::BootstrapHello { + node_id, + protocol_version: PROTOCOL_VERSION, + supported_formats, + preferred_format, + cluster_id, + advertised_endpoint, + max_results, + }, + } + } + + pub(crate) fn bootstrap_ack( + node_id: NodeId, + selected_format: SerializationFormat, + cluster_id: String, + candidates: Vec, + candidate_ttl_ms: u64, + ) -> Self { + Self { + kind: WireMessageKind::BootstrapAck { + node_id, + protocol_version: PROTOCOL_VERSION, + selected_format, + cluster_id, + candidates, + candidate_ttl_ms, + }, + } + } + + pub(crate) fn wire_error(error: ProtocolWireError) -> Self { + Self { + kind: WireMessageKind::Error { error }, + } + } + + pub(crate) fn to_bytes_with_format(&self, format: SerializationFormat) -> NetResult> { + encode_frame( + format, + || serde_json::to_vec(self).map_err(NetError::from), + || { + Ok(wincode::config::serialize( + self, + wincode::config::Configuration::default().disable_preallocation_size_limit(), + )?) + }, + ) + } + + pub(crate) fn from_bytes_with_format(bytes: &[u8]) -> NetResult<(SerializationFormat, Self)> { + decode_frame( + bytes, + |payload| serde_json::from_slice(payload), + deserialize_binary_wire_message, + ) + } +} + +impl From for ProtocolWireError { + fn from(error: WireError) -> Self { + match error { + WireError::ProtocolMismatch { expected, got } => { + Self::ProtocolMismatch { expected, got } + } + WireError::OpRejected { reason } => Self::OpRejected { reason }, + WireError::RateLimited { retry_after_ms } => Self::RateLimited { retry_after_ms }, + WireError::NotAuthorized { reason } => Self::NotAuthorized { reason }, + WireError::Internal { reason } => Self::Internal { reason }, + } + } +} + +impl TryFrom for WireError { + type Error = NetError; + + fn try_from(error: ProtocolWireError) -> Result { + match error { + ProtocolWireError::ProtocolMismatch { expected, got } => { + Ok(Self::ProtocolMismatch { expected, got }) + } + ProtocolWireError::OpRejected { reason } => Ok(Self::OpRejected { reason }), + ProtocolWireError::RateLimited { retry_after_ms } => { + Ok(Self::RateLimited { retry_after_ms }) + } + ProtocolWireError::NotAuthorized { reason } => Ok(Self::NotAuthorized { reason }), + ProtocolWireError::Internal { reason } => Ok(Self::Internal { reason }), + ProtocolWireError::BootstrapRejected { .. } => Err(NetError::InvalidMessage( + "bootstrap wire errors are not public protocol messages".into(), + )), + } + } +} + +impl From for WireMessage { + fn from(message: Message) -> Self { + let kind = match message.kind { + MessageKind::Hello { + node_id, + protocol_version, + supported_formats, + preferred_format, + } => WireMessageKind::Hello { + node_id, + protocol_version, + supported_formats, + preferred_format, + }, + MessageKind::HelloAck { + node_id, + protocol_version, + selected_format, + } => WireMessageKind::HelloAck { + node_id, + protocol_version, + selected_format, + }, + MessageKind::PushOps { ops } => WireMessageKind::PushOps { ops }, + MessageKind::PushOpsAck { received_count } => { + WireMessageKind::PushOpsAck { received_count } + } + MessageKind::PullSince { since_op_id } => WireMessageKind::PullSince { since_op_id }, + MessageKind::Ping => WireMessageKind::Ping, + MessageKind::Pong => WireMessageKind::Pong, + MessageKind::Error { error } => WireMessageKind::Error { + error: error.into(), + }, }; + Self { kind } + } +} - let format = SerializationFormat::from_wire_byte(format_byte)?; - let msg = match format { - SerializationFormat::Json => serde_json::from_slice(payload)?, - SerializationFormat::Bincode => deserialize_binary(payload)?, +impl TryFrom for Message { + type Error = NetError; + + fn try_from(message: WireMessage) -> Result { + let kind = match message.kind { + WireMessageKind::Hello { + node_id, + protocol_version, + supported_formats, + preferred_format, + } => MessageKind::Hello { + node_id, + protocol_version, + supported_formats, + preferred_format, + }, + WireMessageKind::HelloAck { + node_id, + protocol_version, + selected_format, + } => MessageKind::HelloAck { + node_id, + protocol_version, + selected_format, + }, + WireMessageKind::PushOps { ops } => MessageKind::PushOps { ops }, + WireMessageKind::PushOpsAck { received_count } => { + MessageKind::PushOpsAck { received_count } + } + WireMessageKind::PullSince { since_op_id } => MessageKind::PullSince { since_op_id }, + WireMessageKind::Ping => MessageKind::Ping, + WireMessageKind::Pong => MessageKind::Pong, + WireMessageKind::Error { error } => MessageKind::Error { + error: error.try_into()?, + }, + WireMessageKind::BootstrapHello { .. } | WireMessageKind::BootstrapAck { .. } => { + return Err(NetError::InvalidMessage( + "bootstrap wire messages are not public protocol messages".into(), + )); + } }; - Ok((format, msg)) + Ok(Self { kind }) } } +fn encode_frame( + format: SerializationFormat, + json: impl FnOnce() -> NetResult>, + binary: impl FnOnce() -> NetResult>, +) -> NetResult> { + let payload = match format { + SerializationFormat::Json => json()?, + SerializationFormat::Bincode => binary()?, + }; + let len = payload + .len() + .checked_add(1) + .and_then(|len| u32::try_from(len).ok()) + .ok_or_else(|| NetError::InvalidMessage("message payload exceeds u32".to_string()))?; + let mut buffer = Vec::with_capacity(4 + 1 + payload.len()); + buffer.extend_from_slice(&len.to_be_bytes()); + buffer.push(format.to_wire_byte()); + buffer.extend_from_slice(&payload); + Ok(buffer) +} + +fn decode_frame( + bytes: &[u8], + json: impl FnOnce(&[u8]) -> Result, + binary: impl FnOnce(&[u8]) -> Result, +) -> NetResult<(SerializationFormat, T)> { + let Some((&format_byte, payload)) = bytes.split_first() else { + return Err(NetError::InvalidMessage( + "message payload is missing serialization format byte".to_string(), + )); + }; + let format = SerializationFormat::from_wire_byte(format_byte)?; + let message = match format { + SerializationFormat::Json => json(payload)?, + SerializationFormat::Bincode => binary(payload)?, + }; + Ok((format, message)) +} + pub(crate) fn validate_payload_len(len: usize, limit: usize) -> NetResult<()> { if len > limit { return Err(NetError::MessageTooLarge { len, limit }); @@ -332,27 +625,37 @@ fn select_binary_preallocation_limit(declared_len: usize) -> BinaryPreallocation } } -fn deserialize_binary(payload: &[u8]) -> Result { - fn with_limit(payload: &[u8]) -> Result { - wincode::config::deserialize_exact( - payload, - wincode::config::Configuration::default().with_preallocation_size_limit::(), - ) - } +macro_rules! binary_deserializer { + ($name:ident, $message:ty) => { + fn $name(payload: &[u8]) -> Result<$message, wincode::ReadError> { + macro_rules! with_limit { + ($limit:expr) => { + wincode::config::deserialize_exact( + payload, + wincode::config::Configuration::default() + .with_preallocation_size_limit::<$limit>(), + ) + }; + } - match select_binary_preallocation_limit(payload.len()) { - BinaryPreallocationLimit::Limit4MiB => with_limit::<{ 4 * MIB }>(payload), - BinaryPreallocationLimit::Limit16MiB => with_limit::<{ 16 * MIB }>(payload), - BinaryPreallocationLimit::Limit64MiB => with_limit::<{ 64 * MIB }>(payload), - BinaryPreallocationLimit::Limit256MiB => with_limit::<{ 256 * MIB }>(payload), - BinaryPreallocationLimit::Limit1024MiB => with_limit::<{ 1024 * MIB }>(payload), - BinaryPreallocationLimit::Disabled => wincode::config::deserialize_exact( - payload, - wincode::config::Configuration::default().disable_preallocation_size_limit(), - ), - } + match select_binary_preallocation_limit(payload.len()) { + BinaryPreallocationLimit::Limit4MiB => with_limit!({ 4 * MIB }), + BinaryPreallocationLimit::Limit16MiB => with_limit!({ 16 * MIB }), + BinaryPreallocationLimit::Limit64MiB => with_limit!({ 64 * MIB }), + BinaryPreallocationLimit::Limit256MiB => with_limit!({ 256 * MIB }), + BinaryPreallocationLimit::Limit1024MiB => with_limit!({ 1024 * MIB }), + BinaryPreallocationLimit::Disabled => wincode::config::deserialize_exact( + payload, + wincode::config::Configuration::default().disable_preallocation_size_limit(), + ), + } + } + }; } +binary_deserializer!(deserialize_binary_message, Message); +binary_deserializer!(deserialize_binary_wire_message, WireMessage); + #[cfg(test)] mod tests { use super::*; @@ -386,7 +689,7 @@ mod tests { } } - fn protocol_v4_messages() -> Vec { + fn legacy_protocol_v5_messages() -> Vec { let origin = NodeId::new("node-a"); let ops = vec![ Op { @@ -507,6 +810,53 @@ mod tests { ] } + fn bootstrap_wire_messages() -> Vec { + vec![ + WireMessage::wire_error(ProtocolWireError::BootstrapRejected { + reason: "wrong cluster".into(), + }), + WireMessage::bootstrap_hello( + NodeId::new("bootstrap-client"), + DEFAULT_SUPPORTED_FORMATS.to_vec(), + SerializationFormat::Bincode, + "cluster-a".into(), + Some("client.example:9000".into()), + 32, + ), + WireMessage::bootstrap_ack( + NodeId::new("bootstrap-seed"), + SerializationFormat::Bincode, + "cluster-a".into(), + vec!["one.example:9000".into(), "two.example:9001".into()], + 60_000, + ), + ] + } + + fn protocol_v014_fixture_messages() -> Vec { + legacy_protocol_v5_messages() + .into_iter() + .map(|mut message| { + match &mut message.kind { + MessageKind::Hello { + protocol_version, .. + } + | MessageKind::HelloAck { + protocol_version, .. + } => *protocol_version = 4, + MessageKind::Error { + error: WireError::ProtocolMismatch { expected, got }, + } => { + *expected = 4; + *got = 3; + } + _ => {} + } + message + }) + .collect() + } + #[test] fn test_hello_message() { let node_id = NodeId::new("test-node"); @@ -551,6 +901,18 @@ mod tests { assert_eq!(parsed, msg); } + #[test] + fn protocol_v5_messages_roundtrip_in_json() { + for message in legacy_protocol_v5_messages() { + let bytes = message + .to_bytes_with_format(SerializationFormat::Json) + .unwrap(); + let (format, parsed) = Message::from_bytes_with_format(&bytes[4..]).unwrap(); + assert_eq!(format, SerializationFormat::Json); + assert_eq!(parsed, message); + } + } + #[test] fn test_message_roundtrip_bincode() { let node = NodeId::new("node-1"); @@ -569,23 +931,25 @@ mod tests { } #[test] - fn protocol_v4_binary_encoding_matches_bincode_golden_hashes() { + fn protocol_v5_binary_encoding_matches_bincode_golden_hashes() { + // The first thirteen fixtures are the complete public 0.1.4 message + // surface encoded with the intentional protocol-version value 5. let expected_sha256 = [ - "62cb7aa9f8be207d22c1b8e92bdf8096ddc4e1f1ed79a64b7e42047ae267df9a", - "762558e92347d927b302e4a5a22de6a7f61feb74b25108d1adbe0037b93463f8", + "d3cdccc16446588fd57d15139604980cb441667ab5604bd95dbc95de9a222934", + "97cc49ef87c772c7eabb0e5e43fd9737460973e18c7e9ab2009dd5b0e6478ad1", "1953b5c9bfa1929dbe636c27e4e6d504d585c2eba0eb4f61d5a955974b57c31d", "7c16f5631b09eef6cfc2ecdfb0d5336adbaa187c45cf7b6c5e37c4b6dc98158d", "88420266dfd64d604627234a8a6c75cf6477c6fd5505df0d17c59959ae9ce234", "0dd60804260500069dbc38d3b7f3cc4c54ae6952e89b620a9c6d7378705e5b78", "2594b6a92ebfb1c3312deb7d01c015fb95e9fbe9bd7bc6b527af07813ec7b910", "7aa8ca4a02506da9133d8f889678b76f716ce45d02e22fdb7b70a15e56a0eff8", - "4779c171ec57c753c34e20aa6a17595fb121d7bea35261f990213a495ef9cca5", + "aa39a5af59f8c5ce2b32cb8b742ecd8879697b7219a19a82cbeea01e8211bedc", "0239a8fac27cbe2066f549e3ef3bf654f34699e7338f328878dbdb5a956096ee", "169f3c91969ead0a7a678f98088e54519e7c8679ed6d8a5ade85d7a00c718e50", "678ff351757c2bbcba3d3aeb9aa6cef34c34dd07b122817509765742351ec3ab", "574f81f9e34c4b5f8d195759d62c42983380a5a83ddd77cfebe5e7dd84425ae0", ]; - let messages = protocol_v4_messages(); + let messages = legacy_protocol_v5_messages(); assert_eq!(messages.len(), expected_sha256.len()); for (message, expected_hash) in messages.into_iter().zip(expected_sha256) { @@ -598,6 +962,64 @@ mod tests { } } + #[test] + fn public_codec_matches_frozen_v014_bincode_fixtures() { + // Copied from the v0.1.4 release test. Do not regenerate these hashes + // from the current implementation: they guard the legacy discriminants. + let expected_sha256 = [ + "62cb7aa9f8be207d22c1b8e92bdf8096ddc4e1f1ed79a64b7e42047ae267df9a", + "762558e92347d927b302e4a5a22de6a7f61feb74b25108d1adbe0037b93463f8", + "1953b5c9bfa1929dbe636c27e4e6d504d585c2eba0eb4f61d5a955974b57c31d", + "7c16f5631b09eef6cfc2ecdfb0d5336adbaa187c45cf7b6c5e37c4b6dc98158d", + "88420266dfd64d604627234a8a6c75cf6477c6fd5505df0d17c59959ae9ce234", + "0dd60804260500069dbc38d3b7f3cc4c54ae6952e89b620a9c6d7378705e5b78", + "2594b6a92ebfb1c3312deb7d01c015fb95e9fbe9bd7bc6b527af07813ec7b910", + "7aa8ca4a02506da9133d8f889678b76f716ce45d02e22fdb7b70a15e56a0eff8", + "4779c171ec57c753c34e20aa6a17595fb121d7bea35261f990213a495ef9cca5", + "0239a8fac27cbe2066f549e3ef3bf654f34699e7338f328878dbdb5a956096ee", + "169f3c91969ead0a7a678f98088e54519e7c8679ed6d8a5ade85d7a00c718e50", + "678ff351757c2bbcba3d3aeb9aa6cef34c34dd07b122817509765742351ec3ab", + "574f81f9e34c4b5f8d195759d62c42983380a5a83ddd77cfebe5e7dd84425ae0", + ]; + + for (message, expected_hash) in protocol_v014_fixture_messages() + .into_iter() + .zip(expected_sha256) + { + let bytes = wincode::serialize(&message).unwrap(); + let actual_hash = hex::encode(::digest(&bytes)); + assert_eq!(actual_hash, expected_hash, "v0.1.4 message: {message:?}"); + } + } + + #[test] + fn private_wire_codec_matches_public_legacy_codec_in_both_formats() { + for message in legacy_protocol_v5_messages() { + for format in [SerializationFormat::Json, SerializationFormat::Bincode] { + let public = message.to_bytes_with_format(format).unwrap(); + let private = WireMessage::from(message.clone()) + .to_bytes_with_format(format) + .unwrap(); + assert_eq!(private, public, "message: {message:?}, format: {format:?}"); + + let (_, decoded) = WireMessage::from_bytes_with_format(&private[4..]).unwrap(); + assert_eq!(Message::try_from(decoded).unwrap(), message); + } + } + } + + #[test] + fn bootstrap_wire_messages_roundtrip_but_are_not_public_messages() { + for message in bootstrap_wire_messages() { + for format in [SerializationFormat::Json, SerializationFormat::Bincode] { + let bytes = message.to_bytes_with_format(format).unwrap(); + let (_, decoded) = WireMessage::from_bytes_with_format(&bytes[4..]).unwrap(); + assert_eq!(decoded, message); + assert!(Message::from_bytes_with_format(&bytes[4..]).is_err()); + } + } + } + #[test] fn bincode_roundtrip_supports_payloads_above_wincode_default_limit() { let message = Message::push_ops(vec![Op { diff --git a/crates/nx-net/src/node.rs b/crates/nx-net/src/node.rs index 38c5862..bffc479 100644 --- a/crates/nx-net/src/node.rs +++ b/crates/nx-net/src/node.rs @@ -1,21 +1,26 @@ -use std::collections::HashMap; -use std::sync::Arc; +use std::collections::{HashMap, HashSet}; +use std::future::Future; +use std::sync::{Arc, Mutex as StdMutex, Weak}; use std::time::Duration; use nx_sync::{NodeId, Op}; use tokio::io::{AsyncReadExt, AsyncWriteExt, WriteHalf}; use tokio::net::{TcpListener, TcpStream}; use tokio::sync::{Mutex, OwnedSemaphorePermit, RwLock, Semaphore, mpsc, watch}; -use tokio::task::JoinHandle; +use tokio::task::JoinSet; use tokio::time::timeout; use tracing::{debug, error, info, warn}; -use crate::error::{NetError, NetResult}; +use crate::bootstrap::{BootstrapServer, BootstrapServerConfig}; +use crate::error::{BootstrapResult, NetError, NetResult, NodeConfigError}; use crate::message::{ - DEFAULT_SUPPORTED_FORMATS, Message, MessageKind, PROTOCOL_VERSION, SerializationFormat, - WireError, validate_payload_len, + DEFAULT_SUPPORTED_FORMATS, Message, MessageKind, PROTOCOL_VERSION, ProtocolWireError, + SerializationFormat, WireError, WireMessage, WireMessageKind, validate_payload_len, +}; +use crate::peer::{ + ConnectionDirection, PeerConnectionInfo, PeerIdentity, PeerIdentityVerification, PeerInfo, + PeerState, }; -use crate::peer::{PeerInfo, PeerState}; use crate::tls::{NetStream, TlsConfig}; /// Default maximum number of simultaneously connected peers. @@ -32,9 +37,118 @@ pub const DEFAULT_EVENT_CHANNEL_CAPACITY: usize = 1024; /// Time allowed for network tasks to finish cooperatively after shutdown. const TASK_SHUTDOWN_GRACE: Duration = Duration::from_secs(3); +const MAX_CONCURRENT_OUTBOUND_ATTEMPTS: usize = 1; type PeerWriter = Arc>>; +/// Generation identity and cancellation shared by the reader and write snapshots. +struct ConnectionInstance { + closed_tx: watch::Sender, +} + +impl ConnectionInstance { + fn new() -> Self { + Self { + closed_tx: watch::channel(false).0, + } + } +} + +async fn wait_for_stop(mut receiver: watch::Receiver) { + // Do not let a watch guard escape into select! branch outputs (it is not Send). + let _ = receiver.wait_for(|stopped| *stopped).await; +} + +async fn while_connection_open( + closed_rx: &watch::Receiver, + shutdown_rx: &watch::Receiver, + work: impl Future, +) -> Option { + tokio::select! { + biased; + _ = wait_for_stop(closed_rx.clone()) => None, + _ = wait_for_stop(shutdown_rx.clone()) => None, + result = work => Some(result), + } +} + +#[derive(Default)] +struct TaskRegistry { + closed: bool, + tasks: JoinSet<()>, +} + +impl TaskRegistry { + fn spawn(&mut self, task: impl Future + Send + 'static) -> NetResult<()> { + if self.closed { + return Err(NetError::ConnectionFailed("node is shut down".into())); + } + while self.tasks.try_join_next().is_some() {} + // Admission and spawn share the shutdown lock: no untracked task can escape. + self.tasks.spawn(task); + Ok(()) + } +} + +fn spawn_task( + registry: &StdMutex, + task: impl Future + Send + 'static, +) -> NetResult<()> { + registry + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .spawn(task) +} + +/// Owns the drained tasks across awaits. Cancellation aborts them without detaching +/// their handles, so a subsequent shutdown can still join every owned task. +struct ShutdownTasks<'a> { + registry: &'a StdMutex, + tasks: JoinSet<()>, +} + +impl<'a> ShutdownTasks<'a> { + fn close(registry: &'a StdMutex) -> Self { + let mut state = registry + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + state.closed = true; + Self { + registry, + tasks: std::mem::take(&mut state.tasks), + } + } + + async fn join(&mut self, grace: Duration) { + if timeout(grace, async { + while let Some(result) = self.tasks.join_next().await { + if let Err(error) = result { + debug!(%error, "network task ended during shutdown"); + } + } + }) + .await + .is_err() + { + warn!("network tasks did not finish cooperatively; aborting"); + self.tasks.abort_all(); + while self.tasks.join_next().await.is_some() {} + } + } +} + +impl Drop for ShutdownTasks<'_> { + fn drop(&mut self) { + self.tasks.abort_all(); + let mut registry = self + .registry + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + // Shutdown callers are serialized and a closed registry rejects all spawns. + registry.tasks = std::mem::take(&mut self.tasks); + } +} + #[derive(Debug, Clone, Copy)] struct NodeLimits { max_peers: usize, @@ -51,6 +165,7 @@ struct IncomingContext { limits: NodeLimits, slot: OwnedSemaphorePermit, shutdown_rx: watch::Receiver, + bootstrap_server: Option>, } struct ReadLoopContext { @@ -62,6 +177,21 @@ struct ReadLoopContext { max_message_size: usize, socket_timeout: Duration, shutdown_rx: watch::Receiver, + closed_rx: watch::Receiver, +} + +enum IncomingHandshake { + Peer { + node_id: NodeId, + format: SerializationFormat, + }, + Bootstrap { + node_id: NodeId, + format: SerializationFormat, + cluster_id: String, + advertised_endpoint: Option, + max_results: usize, + }, } /// Node configuration. @@ -96,6 +226,30 @@ pub struct NodeConfig { } impl NodeConfig { + /// Check limits before allocating channels or starting network tasks. + /// Zero peers disables admission; event capacity and socket timeout must be positive. + pub fn validate(&self) -> Result<(), NodeConfigError> { + if self.max_peers > Semaphore::MAX_PERMITS { + return Err(NodeConfigError::MaxPeersTooLarge { + limit: Semaphore::MAX_PERMITS, + }); + } + if self.event_channel_capacity == 0 || self.event_channel_capacity > Semaphore::MAX_PERMITS + { + return Err(NodeConfigError::InvalidEventChannelCapacity { + limit: Semaphore::MAX_PERMITS, + }); + } + if self.socket_timeout.is_zero() + || std::time::Instant::now() + .checked_add(self.socket_timeout) + .is_none() + { + return Err(NodeConfigError::InvalidSocketTimeout); + } + Ok(()) + } + pub fn new(node_id: NodeId, listen_addr: impl Into) -> Self { Self { node_id, @@ -180,12 +334,53 @@ pub enum NodeEvent { /// connection; dropping the connection releases capacity. struct PeerConnection { info: PeerInfo, + connection_info: Option, + instance: Arc, state: PeerState, serialization_format: SerializationFormat, writer: Option, _slot: OwnedSemaphorePermit, } +impl Drop for PeerConnection { + fn drop(&mut self) { + // Removal/replacement cancels every user of this generation, even if the + // reader is sending an event or waiting for the writer mutex. + self.instance.closed_tx.send_replace(true); + } +} + +struct ConnectionAttemptGuard { + endpoint: String, + attempts: Arc>>, +} + +impl ConnectionAttemptGuard { + fn acquire(endpoint: &str, attempts: Arc>>) -> NetResult { + let mut active = attempts.lock().map_err(|_| { + NetError::ConnectionFailed("outbound attempt registry is poisoned".to_string()) + })?; + if !active.insert(endpoint.to_string()) { + return Err(NetError::ConnectionFailed(format!( + "connection attempt already in progress for peer: {endpoint}" + ))); + } + drop(active); + Ok(Self { + endpoint: endpoint.to_string(), + attempts, + }) + } +} + +impl Drop for ConnectionAttemptGuard { + fn drop(&mut self) { + if let Ok(mut active) = self.attempts.lock() { + active.remove(&self.endpoint); + } + } +} + /// node pub struct Node { config: NodeConfig, @@ -194,11 +389,15 @@ pub struct Node { event_rx: Option>, shutdown_tx: watch::Sender, connection_slots: Arc, - tasks: Arc>>>, + outbound_attempt_slots: Arc, + outbound_attempts: Arc>>, + tasks: Arc>, + shutdown_lock: Mutex<()>, + bootstrap_server: Option>, } impl Node { - /// crate new node + /// Create a node using the legacy infallible constructor. pub fn new(config: NodeConfig) -> Self { let event_channel_capacity = config.event_channel_capacity.max(1); let (event_tx, event_rx) = mpsc::channel(event_channel_capacity); @@ -212,10 +411,32 @@ impl Node { event_rx: Some(event_rx), shutdown_tx, connection_slots: Arc::new(Semaphore::new(max_peers)), - tasks: Arc::new(Mutex::new(Vec::new())), + outbound_attempt_slots: Arc::new(Semaphore::new(MAX_CONCURRENT_OUTBOUND_ATTEMPTS)), + outbound_attempts: Arc::new(StdMutex::new(HashSet::new())), + tasks: Arc::new(StdMutex::new(TaskRegistry::default())), + shutdown_lock: Mutex::new(()), + bootstrap_server: None, } } + /// Validate configuration and create a node, without binding any sockets. + pub fn try_new(config: NodeConfig) -> Result { + config.validate()?; + Ok(Self::new(config)) + } + + /// Validate node and bootstrap policy, then create a bootstrap-capable node. + pub fn try_new_with_bootstrap_server( + config: NodeConfig, + bootstrap_server: BootstrapServerConfig, + ) -> BootstrapResult { + config.validate()?; + bootstrap_server.validate()?; + let mut node = Self::new(config); + node.bootstrap_server = Some(Arc::new(BootstrapServer::new(bootstrap_server))); + Ok(node) + } + /// Gets the event receiver (can only be called once). pub fn take_event_receiver(&mut self) -> Option> { self.event_rx.take() @@ -244,9 +465,13 @@ impl Node { }; let mut shutdown_rx = self.shutdown_tx.subscribe(); let shutdown_tx = self.shutdown_tx.clone(); + let bootstrap_server = self.bootstrap_server.clone(); - let listener_task = tokio::spawn(async move { + spawn_task(&self.tasks, async move { loop { + if *shutdown_rx.borrow() { + break; + } tokio::select! { _ = shutdown_rx.changed() => { if *shutdown_rx.borrow() { @@ -271,8 +496,9 @@ impl Node { let tls = tls.clone(); let limits = limits; let shutdown_rx = shutdown_tx.subscribe(); + let bootstrap_server = bootstrap_server.clone(); - let task = tokio::spawn(async move { + let admitted = spawn_task(&tasks, async move { let context = IncomingContext { tls, our_node_id: node_id, @@ -281,6 +507,7 @@ impl Node { limits, slot, shutdown_rx, + bootstrap_server, }; if let Err(e) = @@ -289,7 +516,9 @@ impl Node { error!(%addr, error = %e, "connection error"); } }); - track_task(&tasks, task).await; + if admitted.is_err() { + break; + } } Err(e) => { error!(error = %e, "accept error"); @@ -298,46 +527,33 @@ impl Node { } } } - }); - track_task(&self.tasks, listener_task).await; + })?; Ok(bound_addr) } /// Conncet to a peer pub async fn connect_to_peer(&self, addr: &str) -> NetResult<()> { + if *self.shutdown_tx.borrow() { + return Err(NetError::ConnectionFailed("node is shut down".into())); + } + if self.is_connected_addr(addr).await { + return Ok(()); + } + let _attempt = ConnectionAttemptGuard::acquire(addr, Arc::clone(&self.outbound_attempts))?; + let _attempt_slot = Arc::clone(&self.outbound_attempt_slots) + .try_acquire_owned() + .map_err(|_| { + NetError::ConnectionFailed(format!( + "outbound connection attempt limit reached: {MAX_CONCURRENT_OUTBOUND_ATTEMPTS}" + )) + })?; let slot = Arc::clone(&self.connection_slots) .try_acquire_owned() .map_err(|_| NetError::PeerLimitReached(self.config.max_peers))?; - let tcp = timeout(self.config.socket_timeout, TcpStream::connect(addr)) - .await - .map_err(|_| NetError::Timeout)? - .map_err(|e| NetError::ConnectionFailed(format!("{}: {}", addr, e)))?; - - let stream: NetStream = if let Some(tls_cfg) = &self.config.tls { - // Extract host from "host:port" - let host = addr.rsplit_once(':').map(|(h, _)| h).unwrap_or(addr); - - // rustls verifies the presented certificate against this name - // ServerName returned above is not 'static; turn it into owned 'static - let server_name = rustls::pki_types::ServerName::try_from(host) - .or_else(|_| rustls::pki_types::ServerName::try_from("localhost")) - .map_err(|e| { - NetError::TlsError(format!("invalid server name '{}': {}", host, e)) - })?; - - let server_name = server_name.to_owned(); - - timeout( - self.config.socket_timeout, - tls_cfg.connect_stream(tcp, server_name), - ) - .await - .map_err(|_| NetError::Timeout)?? - } else { - NetStream::Plain(tcp) - }; + let (stream, transport_addr) = + connect_transport(addr, self.config.tls.as_ref(), self.config.socket_timeout).await?; // Capture the peer certificate (owned) before moving the stream into split(). let peer_cert = stream.peer_cert_der(); @@ -393,71 +609,51 @@ impl Node { } }; - // TLS identity binding: claimed NodeId must match the peer certificate public key. - if let Some(tls_cfg) = &self.config.tls - && !tls_cfg.insecure - { - let peer_cert = peer_cert.ok_or_else(|| { - NetError::TlsError("missing peer certificate in TLS session".into()) - })?; - - let expected = crate::tls::derive_protocol_node_id_from_cert(&peer_cert)?; - - if peer_node_id != expected { - let fingerprint = crate::tls::cert_fingerprint_hex(&peer_cert) - .unwrap_or_else(|_| "".into()); - - return Err(NetError::TlsError(format!( - "node_id mismatch (claimed={:?}, expected={:?}, fingerprint={})", - peer_node_id, expected, fingerprint - ))); - } - - // Optional allowlist enforcement (permissioned network). - if let Some(_allowed) = &tls_cfg.allowed_peers { - // Peer NodeId on the wire is nx_sync::NodeId; allowlist stores strings. - let peer_id_str = peer_node_id.to_string(); - if !tls_cfg.is_peer_allowed(&peer_id_str) { - return Err(NetError::TlsError(format!( - "peer node_id not in allowlist: {:?}", - peer_node_id - ))); - } - } - } + verify_peer_identity( + &self.config.node_id, + &peer_node_id, + peer_cert.as_ref(), + self.config.tls.as_ref(), + )?; // Save connection let writer = Arc::new(Mutex::new(writer)); + let connection_instance = Arc::new(ConnectionInstance::new()); + let mut peer_connections = self.peers.write().await; let peers_connected = { - let mut peers = self.peers.write().await; - ensure_peer_slot_available(&peers, self.config.max_peers, Some(addr))?; + let peers = &mut *peer_connections; + if peers + .get(addr) + .is_some_and(|connection| connection.state == PeerState::Connected) + { + return Ok(()); + } + ensure_peer_slot_available(peers, self.config.max_peers, Some(addr))?; peers.insert( addr.to_string(), PeerConnection { info: PeerInfo::new(addr).with_node_id(peer_node_id.clone()), + connection_info: Some(PeerConnectionInfo { + transport_addr, + dialed_endpoint: Some(addr.to_string()), + direction: ConnectionDirection::Outbound, + identity: PeerIdentity { + node_id: peer_node_id.clone(), + verification: identity_verification(self.config.tls.as_ref()), + }, + }), + instance: Arc::clone(&connection_instance), state: PeerState::Connected, serialization_format: negotiated_format, writer: Some(Arc::clone(&writer)), _slot: slot, }, ); - connected_peer_count(&peers) + connected_peer_count(peers) }; - let shutdown_for_events = self.shutdown_tx.subscribe(); - send_node_event( - &self.event_tx, - &shutdown_for_events, - "PeerConnected", - NodeEvent::PeerConnected { - node_id: peer_node_id.clone(), - addr: addr.to_string(), - peers_connected, - }, - ) - .await; - - // Start read loop + // No await between inserting the connection and registering its owner. + // Keep the peer lock until rejected admission has rolled the insertion back. let peers = Arc::clone(&self.peers); let event_tx = self.event_tx.clone(); let addr_owned = addr.to_string(); @@ -465,8 +661,28 @@ impl Node { let socket_timeout = self.config.socket_timeout; let shutdown_rx = self.shutdown_tx.subscribe(); let shutdown_for_events = shutdown_rx.clone(); + let task_instance = Arc::clone(&connection_instance); + let (connected_tx, connected_rx) = tokio::sync::oneshot::channel(); + + let admitted = spawn_task(&self.tasks, async move { + let closed_rx = task_instance.closed_tx.subscribe(); + while_connection_open( + &closed_rx, + &shutdown_for_events, + send_node_event( + &event_tx, + &shutdown_for_events, + "PeerConnected", + NodeEvent::PeerConnected { + node_id: peer_node_id.clone(), + addr: addr_owned.clone(), + peers_connected, + }, + ), + ) + .await; + let _ = connected_tx.send(()); - let task = tokio::spawn(async move { if let Err(e) = read_loop( reader, ReadLoopContext { @@ -478,6 +694,7 @@ impl Node { max_message_size, socket_timeout, shutdown_rx, + closed_rx, }, ) .await @@ -488,15 +705,17 @@ impl Node { // Cleanup let disconnected = { let mut peers = peers.write().await; - peers.remove(&addr_owned).and_then(|removed| { - (removed.state == PeerState::Connected).then(|| { - ( - peer_node_id.clone(), - addr_owned.clone(), - connected_peer_count(&peers), - ) - }) - }) + remove_connection_if_current(&mut peers, &addr_owned, &task_instance).and_then( + |removed| { + (removed.state == PeerState::Connected).then(|| { + ( + peer_node_id.clone(), + addr_owned.clone(), + connected_peer_count(&peers), + ) + }) + }, + ) }; if let Some((node_id, addr, peers_connected)) = disconnected { @@ -513,9 +732,16 @@ impl Node { .await; } }); - track_task(&self.tasks, task).await; - - Ok(()) + if admitted.is_err() { + remove_connection_if_current(&mut peer_connections, addr, &connection_instance); + } + drop(peer_connections); + admitted?; + // Preserve event delivery before returning, but keep the read task owned + // even if the caller cancels while the bounded event queue is full. + connected_rx.await.map_err(|_| { + NetError::ConnectionFailed("connection task stopped before announcing the peer".into()) + }) } /// Send ops to all connected peers. @@ -548,7 +774,12 @@ impl Node { (conn.state == PeerState::Connected) .then(|| { conn.writer.as_ref().map(|writer| { - (addr.clone(), Arc::clone(writer), conn.serialization_format) + ( + addr.clone(), + Arc::downgrade(writer), + conn.serialization_format, + Arc::clone(&conn.instance), + ) }) }) .flatten() @@ -557,13 +788,17 @@ impl Node { }; let mut failed = Vec::new(); - for (addr, writer, serialization_format) in writers { + for (addr, writer, serialization_format, instance) in writers { let bytes = msg.to_bytes_with_format(serialization_format)?; - let mut writer = writer.lock().await; - if let Err(e) = write_bytes(&mut *writer, &bytes, self.config.socket_timeout).await { + if let Err(e) = self + .write_to_connection(&addr, writer, &instance, &bytes) + .await + { warn!(%addr, error = %e, "failed to send ops"); failed.push(addr.clone()); - if let Some((node_id, peers_connected)) = self.mark_peer_failed(&addr).await { + if let Some((node_id, peers_connected)) = + self.mark_peer_failed(&addr, &instance).await + { let shutdown_for_events = self.shutdown_tx.subscribe(); send_node_event( &self.event_tx, @@ -593,23 +828,29 @@ impl Node { peers.get(addr).and_then(|conn| { (conn.state == PeerState::Connected) .then(|| { - conn.writer - .as_ref() - .map(|writer| (Arc::clone(writer), conn.serialization_format)) + conn.writer.as_ref().map(|writer| { + ( + Arc::downgrade(writer), + conn.serialization_format, + Arc::clone(&conn.instance), + ) + }) }) .flatten() }) }; - let Some((writer, serialization_format)) = peer_writer else { + let Some((writer, serialization_format, instance)) = peer_writer else { return Err(NetError::PeerDisconnected(addr.to_string())); }; let bytes = msg.to_bytes_with_format(serialization_format)?; - let mut writer = writer.lock().await; - if let Err(e) = write_bytes(&mut *writer, &bytes, self.config.socket_timeout).await { + if let Err(e) = self + .write_to_connection(addr, writer, &instance, &bytes) + .await + { warn!(%addr, error = %e, "failed to send message to peer"); - if let Some((node_id, peers_connected)) = self.mark_peer_failed(addr).await { + if let Some((node_id, peers_connected)) = self.mark_peer_failed(addr, &instance).await { let shutdown_for_events = self.shutdown_tx.subscribe(); send_node_event( &self.event_tx, @@ -629,12 +870,51 @@ impl Node { Ok(()) } + async fn write_to_connection( + &self, + addr: &str, + writer: Weak>>, + instance: &ConnectionInstance, + bytes: &[u8], + ) -> NetResult<()> { + let closed_rx = instance.closed_tx.subscribe(); + let shutdown_rx = self.shutdown_tx.subscribe(); + while_connection_open(&closed_rx, &shutdown_rx, async move { + let writer = writer + .upgrade() + .ok_or_else(|| NetError::PeerDisconnected(addr.into()))?; + let mut writer = writer.lock().await; + write_bytes(&mut *writer, bytes, self.config.socket_timeout).await + }) + .await + .unwrap_or_else(|| Err(NetError::PeerDisconnected(addr.into()))) + } + /// Returns the number of currently connected peers. pub async fn connected_peer_count(&self) -> usize { let peers = self.peers.read().await; connected_peer_count(&peers) } + /// Snapshot of active peers as `(connection address, handshake NodeId)` pairs. + /// Sorted by address; identities are certificate-bound only with secure TLS. + pub async fn connected_peers(&self) -> Vec<(String, NodeId)> { + let peers = self.peers.read().await; + let mut connected = peers + .iter() + .filter(|(_, connection)| connection.state == PeerState::Connected) + .filter_map(|(addr, connection)| { + connection + .info + .node_id + .clone() + .map(|node_id| (addr.clone(), node_id)) + }) + .collect::>(); + connected.sort_by(|(left, _), (right, _)| left.cmp(right)); + connected + } + /// Returns true when the configured peer address currently has an active connection. pub async fn is_connected_addr(&self, addr: &str) -> bool { let peers = self.peers.read().await; @@ -643,39 +923,63 @@ impl Node { .is_some_and(|conn| conn.state == PeerState::Connected) } - async fn mark_peer_failed(&self, addr: &str) -> Option<(NodeId, usize)> { - let mut peers = self.peers.write().await; - let node_id = { - let conn = peers.get_mut(addr)?; - conn.state = PeerState::Failed; - conn.info.node_id.clone()? - }; - Some((node_id, connected_peer_count(&peers))) + /// Returns authenticated/claimed identity and transport facts for an active connection. + pub async fn connection_info(&self, addr: &str) -> Option { + let peers = self.peers.read().await; + peers.get(addr).and_then(|connection| { + (connection.state == PeerState::Connected) + .then(|| connection.connection_info.clone()) + .flatten() + }) } - /// Close outbound peer connections by dropping their writers. - pub async fn shutdown(&self) { - let _ = self.shutdown_tx.send(true); + /// Publish the endpoint returned by this node's bootstrap service. + pub fn announce_bootstrap_endpoint(&self, endpoint: impl Into) -> NetResult<()> { + let server = self + .bootstrap_server + .as_ref() + .ok_or_else(|| NetError::InvalidMessage("bootstrap server is not configured".into()))?; + server.announce(endpoint.into()) + } - let mut tasks = { - let mut tasks = self.tasks.lock().await; - std::mem::take(&mut *tasks) - }; + /// Withdraw the endpoint returned by this node's bootstrap service. + pub fn withdraw_bootstrap_endpoint(&self) -> NetResult<()> { + let server = self + .bootstrap_server + .as_ref() + .ok_or_else(|| NetError::InvalidMessage("bootstrap server is not configured".into()))?; + server.withdraw() + } - for mut task in tasks.drain(..) { - match timeout(TASK_SHUTDOWN_GRACE, &mut task).await { - Ok(Ok(())) => {} - Ok(Err(e)) => { - debug!(error = %e, "network task ended during shutdown"); - } - Err(_) => { - warn!("network task did not finish cooperatively; aborting"); - task.abort(); - let _ = task.await; - } - } + async fn mark_peer_failed( + &self, + addr: &str, + instance: &Arc, + ) -> Option<(NodeId, usize)> { + let mut peers = self.peers.write().await; + let removed = remove_connection_if_current(&mut peers, addr, instance)?; + let node_id = (removed.state == PeerState::Connected) + .then(|| removed.info.node_id.clone()) + .flatten(); + // Release admission and signal cancellation before any event queue await. + drop(removed); + node_id.map(|node_id| (node_id, connected_peer_count(&peers))) + } + + /// Stop admissions, join owned network tasks within one grace period, then + /// abort/join any stragglers and close peer writers. Safe to retry if cancelled. + pub async fn shutdown(&self) { + let _shutdown = self.shutdown_lock.lock().await; + let mut tasks = ShutdownTasks::close(&self.tasks); + self.shutdown_tx.send_replace(true); + self.connection_slots.close(); + self.outbound_attempt_slots.close(); + if let Some(server) = &self.bootstrap_server { + server.clear(); } + tasks.join(TASK_SHUTDOWN_GRACE).await; + let count = { let mut peers = self.peers.write().await; let count = peers.len(); @@ -686,6 +990,23 @@ impl Node { } } +impl Drop for Node { + fn drop(&mut self) { + let mut registry = self + .tasks + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + registry.closed = true; + registry.tasks.abort_all(); + self.shutdown_tx.send_replace(true); + self.connection_slots.close(); + self.outbound_attempt_slots.close(); + if let Some(server) = &self.bootstrap_server { + server.clear(); + } + } +} + /// Manage an incoming connection from a peer (handshake + read loop). async fn handle_incoming( stream: TcpStream, @@ -700,6 +1021,7 @@ async fn handle_incoming( limits, slot, shutdown_rx, + bootstrap_server, } = context; let stream: NetStream = match tls { @@ -716,10 +1038,10 @@ async fn handle_incoming( // Wait for HELLO let (hello_format, msg) = - read_message_with_format(&mut reader, limits.max_message_size, limits.socket_timeout) + read_wire_message_with_format(&mut reader, limits.max_message_size, limits.socket_timeout) .await?; - let (peer_node_id, negotiated_format) = match msg.kind { - MessageKind::Hello { + let handshake = match msg.kind { + WireMessageKind::Hello { node_id, protocol_version, supported_formats, @@ -729,7 +1051,7 @@ async fn handle_incoming( let error = WireError::protocol_mismatch(protocol_version); let _ = write_message( &mut writer, - &Message::wire_error(error), + &WireMessage::wire_error(error.into()), hello_format, limits.socket_timeout, ) @@ -751,47 +1073,158 @@ async fn handle_incoming( "selected alternate serialization format" ); } - (node_id, negotiated_format) + IncomingHandshake::Peer { + node_id, + format: negotiated_format, + } } - MessageKind::Error { error } => { - return Err(NetError::Wire(error)); + WireMessageKind::BootstrapHello { + node_id, + protocol_version, + supported_formats, + preferred_format, + cluster_id, + advertised_endpoint, + max_results, + } => { + if !is_protocol_version_compatible(protocol_version) { + let error = WireError::protocol_mismatch(protocol_version); + let _ = write_message( + &mut writer, + &WireMessage::wire_error(error.into()), + hello_format, + limits.socket_timeout, + ) + .await; + return Err(protocol_version_mismatch(protocol_version)); + } + let negotiated_format = + negotiate_serialization_format(limits.serialization_format, &supported_formats) + .ok_or_else(|| { + NetError::InvalidMessage( + "no mutually supported serialization format".to_string(), + ) + })?; + if negotiated_format != preferred_format { + debug!( + peer = %node_id, + peer_preferred_format = ?preferred_format, + selected_format = ?negotiated_format, + "selected alternate bootstrap serialization format" + ); + } + IncomingHandshake::Bootstrap { + node_id, + format: negotiated_format, + cluster_id, + advertised_endpoint, + max_results: max_results as usize, + } + } + WireMessageKind::Error { error } => { + return Err(NetError::Wire(error.try_into()?)); } _ => { - return Err(NetError::InvalidMessage("expected Hello".into())); + return Err(NetError::InvalidMessage( + "expected Hello or BootstrapHello".into(), + )); } }; - // TLS identity binding: claimed NodeId must match the peer certificate public key. - if let Some(tls_cfg) = &tls - && !tls_cfg.insecure - { - let peer_cert = peer_cert - .ok_or_else(|| NetError::TlsError("missing peer certificate in TLS session".into()))?; - - let expected = crate::tls::derive_protocol_node_id_from_cert(&peer_cert)?; - - if peer_node_id != expected { - let fingerprint = crate::tls::cert_fingerprint_hex(&peer_cert) - .unwrap_or_else(|_| "".into()); - - return Err(NetError::TlsError(format!( - "node_id mismatch (claimed={:?}, expected={:?}, fingerprint={})", - peer_node_id, expected, fingerprint - ))); + let peer_node_id = match &handshake { + IncomingHandshake::Peer { node_id, .. } | IncomingHandshake::Bootstrap { node_id, .. } => { + node_id } - - // Optional allowlist enforcement (permissioned network). - if let Some(_allowed) = &tls_cfg.allowed_peers { - let peer_id_str = peer_node_id.to_string(); - if !tls_cfg.is_peer_allowed(&peer_id_str) { - return Err(NetError::TlsError(format!( - "peer node_id not in allowlist: {:?}", - peer_node_id - ))); + }; + verify_peer_identity(&our_node_id, peer_node_id, peer_cert.as_ref(), tls.as_ref())?; + + if let IncomingHandshake::Bootstrap { + node_id, + format, + cluster_id, + advertised_endpoint, + max_results, + } = handshake + { + let server = match bootstrap_server { + Some(server) => server, + None => { + let error = ProtocolWireError::BootstrapRejected { + reason: "bootstrap service is disabled".into(), + }; + let _ = write_message( + &mut writer, + &WireMessage::wire_error(error.clone()), + format, + limits.socket_timeout, + ) + .await; + return Err(NetError::InvalidMessage(error.to_string())); } + }; + if cluster_id != server.cluster_id() { + let error = ProtocolWireError::BootstrapRejected { + reason: "cluster ID does not match this bootstrap seed".into(), + }; + let _ = write_message( + &mut writer, + &WireMessage::wire_error(error.clone()), + format, + limits.socket_timeout, + ) + .await; + return Err(NetError::InvalidMessage(error.to_string())); } + if max_results == 0 { + let error = ProtocolWireError::BootstrapRejected { + reason: "max_results must be greater than zero".into(), + }; + let _ = write_message( + &mut writer, + &WireMessage::wire_error(error.clone()), + format, + limits.socket_timeout, + ) + .await; + return Err(NetError::InvalidMessage(error.to_string())); + } + let candidates = match server.exchange(&node_id, advertised_endpoint, max_results) { + Ok(candidates) => candidates, + Err(error) => { + let wire_error = ProtocolWireError::BootstrapRejected { + reason: error.to_string(), + }; + let _ = write_message( + &mut writer, + &WireMessage::wire_error(wire_error.clone()), + format, + limits.socket_timeout, + ) + .await; + return Err(NetError::InvalidMessage(wire_error.to_string())); + } + }; + let candidate_ttl_ms = + u64::try_from(server.candidate_ttl().as_millis()).unwrap_or(u64::MAX); + let ack = WireMessage::bootstrap_ack( + our_node_id, + format, + cluster_id, + candidates, + candidate_ttl_ms, + ); + write_message(&mut writer, &ack, format, limits.socket_timeout).await?; + return Ok(()); } + let IncomingHandshake::Peer { + node_id: peer_node_id, + format: negotiated_format, + } = handshake + else { + unreachable!("bootstrap handshakes return before peer admission") + }; + { let peers = peers.read().await; ensure_peer_slot_available(&peers, limits.max_peers, Some(&addr))?; @@ -801,6 +1234,7 @@ async fn handle_incoming( write_message(&mut writer, &ack, negotiated_format, limits.socket_timeout).await?; let writer = Arc::new(Mutex::new(writer)); + let connection_instance = Arc::new(ConnectionInstance::new()); let peers_connected = { let mut peers = peers.write().await; ensure_peer_slot_available(&peers, limits.max_peers, Some(&addr))?; @@ -808,6 +1242,16 @@ async fn handle_incoming( addr.clone(), PeerConnection { info: PeerInfo::new(&addr).with_node_id(peer_node_id.clone()), + connection_info: Some(PeerConnectionInfo { + transport_addr: addr.clone(), + dialed_endpoint: None, + direction: ConnectionDirection::Inbound, + identity: PeerIdentity { + node_id: peer_node_id.clone(), + verification: identity_verification(tls.as_ref()), + }, + }), + instance: Arc::clone(&connection_instance), state: PeerState::Connected, serialization_format: negotiated_format, writer: Some(Arc::clone(&writer)), @@ -820,15 +1264,20 @@ async fn handle_incoming( info!(peer = %peer_node_id, serialization_format = ?negotiated_format, "incoming peer connected"); let shutdown_for_events = shutdown_rx.clone(); - send_node_event( - &event_tx, + let closed_rx = connection_instance.closed_tx.subscribe(); + while_connection_open( + &closed_rx, &shutdown_for_events, - "PeerConnected", - NodeEvent::PeerConnected { - node_id: peer_node_id.clone(), - addr: addr.clone(), - peers_connected, - }, + send_node_event( + &event_tx, + &shutdown_for_events, + "PeerConnected", + NodeEvent::PeerConnected { + node_id: peer_node_id.clone(), + addr: addr.clone(), + peers_connected, + }, + ), ) .await; @@ -844,13 +1293,15 @@ async fn handle_incoming( max_message_size: limits.max_message_size, socket_timeout: limits.socket_timeout, shutdown_rx, + closed_rx, }, ) .await; let disconnected = { let mut peers = peers.write().await; - let Some(removed) = peers.remove(&addr) else { + let Some(removed) = remove_connection_if_current(&mut peers, &addr, &connection_instance) + else { return read_result; }; (removed.state == PeerState::Connected).then(|| { @@ -876,14 +1327,107 @@ async fn handle_incoming( .await; } - read_result + read_result +} + +fn connected_peer_count(peers: &HashMap) -> usize { + peers + .values() + .filter(|c| c.state == PeerState::Connected) + .count() +} + +fn remove_connection_if_current( + peers: &mut HashMap, + addr: &str, + instance: &Arc, +) -> Option { + peers + .get(addr) + .is_some_and(|connection| Arc::ptr_eq(&connection.instance, instance)) + .then(|| peers.remove(addr)) + .flatten() +} + +pub(crate) async fn connect_transport( + addr: &str, + tls: Option<&TlsConfig>, + socket_timeout: Duration, +) -> NetResult<(NetStream, String)> { + let tcp = timeout(socket_timeout, TcpStream::connect(addr)) + .await + .map_err(|_| NetError::Timeout)? + .map_err(|error| NetError::ConnectionFailed(format!("{addr}: {error}")))?; + let transport_addr = tcp.peer_addr()?.to_string(); + let stream = if let Some(tls_config) = tls { + let host = endpoint_host(addr)?; + let server_name = + rustls::pki_types::ServerName::try_from(host.to_string()).map_err(|error| { + NetError::TlsError(format!("invalid server name '{host}': {error}")) + })?; + timeout(socket_timeout, tls_config.connect_stream(tcp, server_name)) + .await + .map_err(|_| NetError::Timeout)?? + } else { + NetStream::Plain(tcp) + }; + Ok((stream, transport_addr)) +} + +fn endpoint_host(endpoint: &str) -> NetResult<&str> { + if endpoint.starts_with('[') { + let closing = endpoint + .find(']') + .ok_or_else(|| NetError::InvalidMessage("invalid bracketed peer endpoint".into()))?; + return Ok(&endpoint[1..closing]); + } + endpoint + .rsplit_once(':') + .map(|(host, _)| host) + .filter(|host| !host.is_empty()) + .ok_or_else(|| NetError::InvalidMessage("peer endpoint must be host:port".into())) +} + +pub(crate) fn verify_peer_identity( + our_node_id: &NodeId, + peer_node_id: &NodeId, + peer_cert: Option<&rustls::pki_types::CertificateDer<'static>>, + tls: Option<&TlsConfig>, +) -> NetResult<()> { + if peer_node_id == our_node_id { + return Err(NetError::ConnectionFailed(format!( + "refusing connection to local node ID: {peer_node_id}" + ))); + } + + if let Some(tls_config) = tls + && !tls_config.insecure + { + let peer_cert = peer_cert + .ok_or_else(|| NetError::TlsError("missing peer certificate in TLS session".into()))?; + let expected = crate::tls::derive_protocol_node_id_from_cert(peer_cert)?; + if peer_node_id != &expected { + let fingerprint = crate::tls::cert_fingerprint_hex(peer_cert) + .unwrap_or_else(|_| "".into()); + return Err(NetError::TlsError(format!( + "node_id mismatch (claimed={peer_node_id:?}, expected={expected:?}, fingerprint={fingerprint})" + ))); + } + if !tls_config.is_peer_allowed(&peer_node_id.to_string()) { + return Err(NetError::TlsError(format!( + "peer node_id not in allowlist: {peer_node_id:?}" + ))); + } + } + Ok(()) } -fn connected_peer_count(peers: &HashMap) -> usize { - peers - .values() - .filter(|c| c.state == PeerState::Connected) - .count() +fn identity_verification(tls: Option<&TlsConfig>) -> PeerIdentityVerification { + if tls.is_some_and(|config| !config.insecure) { + PeerIdentityVerification::CertificateBound + } else { + PeerIdentityVerification::Unverified + } } fn ensure_peer_slot_available( @@ -903,14 +1447,14 @@ fn ensure_peer_slot_available( Ok(()) } -fn supported_formats_for(preferred: SerializationFormat) -> Vec { +pub(crate) fn supported_formats_for(preferred: SerializationFormat) -> Vec { match preferred { SerializationFormat::Json => vec![SerializationFormat::Json], SerializationFormat::Bincode => DEFAULT_SUPPORTED_FORMATS.to_vec(), } } -fn negotiate_serialization_format( +pub(crate) fn negotiate_serialization_format( preferred: SerializationFormat, peer_supported: &[SerializationFormat], ) -> Option { @@ -937,7 +1481,20 @@ async fn send_node_event( event_name: &'static str, event: NodeEvent, ) { - if let Err(e) = event_tx.send(event).await { + // Keep immediately available shutdown notifications, but never wait for a + // full queue during shutdown. Disconnect callers release peer resources first. + let result = match event_tx.try_send(event) { + Ok(()) => return, + Err(mpsc::error::TrySendError::Closed(event)) => Err(mpsc::error::SendError(event)), + Err(mpsc::error::TrySendError::Full(event)) => { + tokio::select! { + biased; + _ = wait_for_stop(shutdown_rx.clone()) => return, + result = event_tx.send(event) => result, + } + } + }; + if let Err(e) = result { if *shutdown_rx.borrow() { debug!( event = event_name, @@ -950,14 +1507,20 @@ async fn send_node_event( } } -async fn track_task(tasks: &Arc>>>, task: JoinHandle<()>) { - let mut tasks = tasks.lock().await; - tasks.retain(|task| !task.is_finished()); - tasks.push(task); -} - /// Loop for reading messages from a peer until disconnection async fn read_loop( + reader: tokio::io::ReadHalf, + context: ReadLoopContext, +) -> NetResult<()> { + let closed_rx = context.closed_rx.clone(); + let shutdown_rx = context.shutdown_rx.clone(); + // Cover the whole loop, including event backpressure and Ping writer locks. + while_connection_open(&closed_rx, &shutdown_rx, read_messages(reader, context)) + .await + .unwrap_or(Ok(())) +} + +async fn read_messages( mut reader: tokio::io::ReadHalf, context: ReadLoopContext, ) -> NetResult<()> { @@ -970,10 +1533,14 @@ async fn read_loop( max_message_size, socket_timeout, mut shutdown_rx, + closed_rx: _, } = context; let shutdown_for_events = shutdown_rx.clone(); loop { + if *shutdown_rx.borrow() { + break; + } let msg = tokio::select! { _ = shutdown_rx.changed() => { if *shutdown_rx.borrow() { @@ -1050,13 +1617,33 @@ async fn read_loop( } /// Writes a message to a stream. -async fn write_message( +pub(crate) trait WireEncode { + fn encode_wire(&self, format: SerializationFormat) -> NetResult>; +} + +impl WireEncode for Message { + fn encode_wire(&self, format: SerializationFormat) -> NetResult> { + self.to_bytes_with_format(format) + } +} + +impl WireEncode for WireMessage { + fn encode_wire(&self, format: SerializationFormat) -> NetResult> { + self.to_bytes_with_format(format) + } +} + +pub(crate) async fn write_message( writer: &mut W, - msg: &Message, + msg: &M, serialization_format: SerializationFormat, socket_timeout: Duration, -) -> NetResult<()> { - let bytes = msg.to_bytes_with_format(serialization_format)?; +) -> NetResult<()> +where + W: AsyncWriteExt + Unpin, + M: WireEncode, +{ + let bytes = msg.encode_wire(serialization_format)?; write_bytes(writer, &bytes, socket_timeout).await?; Ok(()) } @@ -1076,7 +1663,7 @@ async fn write_bytes( } /// Reads a message from a stream. -async fn read_message( +pub(crate) async fn read_message( reader: &mut R, max_message_size: usize, socket_timeout: Duration, @@ -1085,11 +1672,21 @@ async fn read_message( Ok(msg) } -async fn read_message_with_format( +pub(crate) async fn read_message_with_format( reader: &mut R, max_message_size: usize, socket_timeout: Duration, ) -> NetResult<(SerializationFormat, Message)> { + let (format, message) = + read_wire_message_with_format(reader, max_message_size, socket_timeout).await?; + Ok((format, message.try_into()?)) +} + +pub(crate) async fn read_wire_message_with_format( + reader: &mut R, + max_message_size: usize, + socket_timeout: Duration, +) -> NetResult<(SerializationFormat, WireMessage)> { // Read length (4 bytes) let mut len_buf = [0u8; 4]; timeout(socket_timeout, reader.read_exact(&mut len_buf)) @@ -1105,7 +1702,7 @@ async fn read_message_with_format( .await .map_err(|_| NetError::Timeout)??; - Message::from_bytes_with_format(&buf) + WireMessage::from_bytes_with_format(&buf) } /// Fuzzing-only entry point for the production stream framing path. @@ -1128,6 +1725,306 @@ mod tests { Arc::new(Semaphore::new(1)).try_acquire_owned().unwrap() } + #[test] + fn node_config_validates_semaphore_boundaries_without_allocating_them() { + let config = NodeConfig::new(NodeId::new("test"), "127.0.0.1:0"); + config.clone().with_max_peers(0).validate().unwrap(); + config + .clone() + .with_max_peers(Semaphore::MAX_PERMITS) + .validate() + .unwrap(); + config + .clone() + .with_event_channel_capacity(Semaphore::MAX_PERMITS) + .validate() + .unwrap(); + for limit in [Semaphore::MAX_PERMITS + 1, usize::MAX] { + assert!(matches!( + config.clone().with_max_peers(limit).validate(), + Err(NodeConfigError::MaxPeersTooLarge { .. }) + )); + assert!(matches!( + config.clone().with_event_channel_capacity(limit).validate(), + Err(NodeConfigError::InvalidEventChannelCapacity { .. }) + )); + } + let node = Node::try_new(config).unwrap(); + assert_eq!(node.connection_slots.available_permits(), DEFAULT_MAX_PEERS); + assert_eq!(node.event_tx.max_capacity(), DEFAULT_EVENT_CHANNEL_CAPACITY); + } + + #[tokio::test] + async fn strict_constructor_rejects_invalid_node_configs() { + let config = NodeConfig::new(NodeId::new("test"), "invalid listen address"); + for invalid in [ + config.clone().with_max_peers(Semaphore::MAX_PERMITS + 1), + config.clone().with_max_peers(usize::MAX), + config + .clone() + .with_event_channel_capacity(Semaphore::MAX_PERMITS + 1), + config.clone().with_event_channel_capacity(usize::MAX), + config.clone().with_event_channel_capacity(0), + config.clone().with_socket_timeout(Duration::MAX), + config.clone().with_socket_timeout(Duration::ZERO), + ] { + assert!(Node::try_new(invalid).is_err()); + } + + // Preserve the 0.1.4 constructor behavior for existing callers: a zero + // event capacity is clamped to one by the infallible legacy path. + let node = Node::new(config.with_event_channel_capacity(0)); + assert_eq!(node.event_tx.max_capacity(), 1); + assert_eq!(node.connection_slots.available_permits(), DEFAULT_MAX_PEERS); + node.shutdown().await; + } + + async fn open_raw_peer( + node: &Node, + listener: &TcpListener, + incoming_addr: Option, + ) -> TcpStream { + timeout(Duration::from_secs(3), async { + if let Some(addr) = incoming_addr { + let mut stream = TcpStream::connect(addr).await.unwrap(); + write_message( + &mut stream, + &Message::hello(NodeId::new("raw-peer")), + SerializationFormat::Bincode, + Duration::from_secs(1), + ) + .await + .unwrap(); + let ack = read_message( + &mut stream, + DEFAULT_MAX_MESSAGE_SIZE, + Duration::from_secs(1), + ) + .await + .unwrap(); + assert!(matches!(ack.kind, MessageKind::HelloAck { .. })); + stream + } else { + let addr = listener.local_addr().unwrap().to_string(); + let (connected, stream) = tokio::join!(node.connect_to_peer(&addr), async { + let (mut stream, _) = listener.accept().await.unwrap(); + let hello = read_message( + &mut stream, + DEFAULT_MAX_MESSAGE_SIZE, + Duration::from_secs(1), + ) + .await + .unwrap(); + assert!(matches!(hello.kind, MessageKind::Hello { .. })); + write_message( + &mut stream, + &Message::hello_ack_with_format( + NodeId::new("raw-peer"), + SerializationFormat::Bincode, + ), + SerializationFormat::Bincode, + Duration::from_secs(1), + ) + .await + .unwrap(); + stream + }); + connected.unwrap(); + stream + } + }) + .await + .unwrap() + } + + #[tokio::test] + async fn write_failure_releases_live_reader_and_slot_before_disconnect_event_delivery() { + for incoming in [false, true] { + for broadcast in [false, true] { + let mut node = Node::try_new( + NodeConfig::new(NodeId::new("test"), "127.0.0.1:0") + .with_max_peers(1) + .with_event_channel_capacity(1) + .with_socket_timeout(Duration::from_secs(60)), + ) + .unwrap(); + let mut events = node.take_event_receiver().unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let incoming_addr = if incoming { + Some(node.start_listener().await.unwrap()) + } else { + None + }; + let mut remote = open_raw_peer(&node, &listener, incoming_addr).await; + let Some(NodeEvent::PeerConnected { addr, .. }) = events.recv().await else { + panic!("missing connection event") + }; + let (writer, instance) = { + let peers = node.peers.read().await; + let peer = &peers[&addr]; + ( + Arc::downgrade(peer.writer.as_ref().unwrap()), + Arc::clone(&peer.instance), + ) + }; + assert_eq!(node.connection_slots.available_permits(), 0); + + // Inject a deterministic write-side failure without closing the reader. + // The remote sends a request and never reads again after the handshake. + writer + .upgrade() + .unwrap() + .lock() + .await + .shutdown() + .await + .unwrap(); + write_message( + &mut remote, + &Message::pull_since(None), + SerializationFormat::Bincode, + Duration::from_secs(1), + ) + .await + .unwrap(); + assert!(matches!( + timeout(Duration::from_secs(1), events.recv()) + .await + .unwrap(), + Some(NodeEvent::PullRequested { .. }) + )); + node.event_tx + .try_send(NodeEvent::OpsReceived { + from: NodeId::new("queued"), + ops: vec![], + }) + .unwrap(); + + let mut send = tokio_test::task::spawn(async { + if broadcast { + node.broadcast_message(Message::ping()).await + } else { + node.send_message_to_addr(&addr, Message::ping()).await + } + }); + // The write has failed and only the bounded disconnect event is blocked. + assert!(send.poll().is_pending()); + assert_eq!(node.connection_slots.available_permits(), 1); + assert!(!node.is_connected_addr(&addr).await); + assert!(*instance.closed_tx.borrow()); + timeout(Duration::from_secs(1), async { + while writer.upgrade().is_some() { + tokio::task::yield_now().await; + } + }) + .await + .expect("reader or failed send retained the socket writer"); + assert!(matches!( + events.recv().await, + Some(NodeEvent::OpsReceived { .. }) + )); + assert!(send.await.is_err()); + assert!(matches!( + events.recv().await, + Some(NodeEvent::PeerDisconnected { + peers_connected: 0, + .. + }) + )); + assert!(events.try_recv().is_err(), "duplicate disconnect event"); + + // This must actually acquire the released permit and complete a handshake. + let _replacement = open_raw_peer(&node, &listener, incoming_addr).await; + assert!(matches!( + events.recv().await, + Some(NodeEvent::PeerConnected { + peers_connected: 1, + .. + }) + )); + assert_eq!(node.connection_slots.available_permits(), 0); + assert_eq!(node.connected_peer_count().await, 1); + assert!(node.mark_peer_failed(&addr, &instance).await.is_none()); + assert!(events.try_recv().is_err()); + node.shutdown().await; + } + } + } + + #[tokio::test] + async fn read_loop_cancels_event_backpressure_and_ping_writer_lock() { + for blocked_on_ping in [false, true] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let stream = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (mut remote, _) = listener.accept().await.unwrap(); + let (reader, writer) = tokio::io::split(NetStream::Plain(stream)); + let writer = Arc::new(Mutex::new(writer)); + let held_writer = writer.lock().await; + let instance = ConnectionInstance::new(); + let (shutdown_tx, shutdown_rx) = watch::channel(false); + let (event_tx, mut events) = mpsc::channel(1); + // The first event is an observable barrier; the next frame blocks in + // either event delivery or Ping's writer lock, not in socket reading. + let mut bytes = Message::pull_since(None).to_bytes().unwrap(); + bytes.extend( + if blocked_on_ping { + Message::ping() + } else { + Message::pull_since(Some("blocked".into())) + } + .to_bytes() + .unwrap(), + ); + remote.write_all(&bytes).await.unwrap(); + let mut read = tokio_test::task::spawn(read_loop( + reader, + ReadLoopContext { + peer_node_id: NodeId::new("peer"), + addr: "peer".into(), + event_tx, + writer: Arc::clone(&writer), + serialization_format: SerializationFormat::Bincode, + max_message_size: DEFAULT_MAX_MESSAGE_SIZE, + socket_timeout: Duration::from_secs(60), + shutdown_rx, + closed_rx: instance.closed_tx.subscribe(), + }, + )); + timeout( + Duration::from_secs(1), + std::future::poll_fn(|cx| { + assert!(read.poll().is_pending()); + if events.len() == 1 { + std::task::Poll::Ready(()) + } else { + cx.waker().wake_by_ref(); + std::task::Poll::Pending + } + }), + ) + .await + .unwrap(); + instance.closed_tx.send_replace(true); + timeout(Duration::from_secs(1), read) + .await + .unwrap() + .unwrap(); + assert_eq!(Arc::strong_count(&writer), 1, "reader retained its writer"); + assert!(matches!( + events.try_recv(), + Ok(NodeEvent::PullRequested { + since_op_id: None, + .. + }) + )); + assert!(events.try_recv().is_err()); + drop(held_writer); + drop(shutdown_tx); + } + } + #[tokio::test] async fn test_node_config() { let config = NodeConfig::new(NodeId::new("test"), "127.0.0.1:9000") @@ -1185,6 +2082,8 @@ mod tests { "127.0.0.1:9001".to_string(), PeerConnection { info: PeerInfo::new("127.0.0.1:9001"), + connection_info: None, + instance: Arc::new(ConnectionInstance::new()), state: PeerState::Connected, serialization_format: SerializationFormat::Bincode, writer: None, @@ -1204,6 +2103,8 @@ mod tests { "127.0.0.1:9001".to_string(), PeerConnection { info: PeerInfo::new("127.0.0.1:9001"), + connection_info: None, + instance: Arc::new(ConnectionInstance::new()), state: PeerState::Connected, serialization_format: SerializationFormat::Bincode, writer: None, @@ -1214,6 +2115,30 @@ mod tests { ensure_peer_slot_available(&peers, 1, Some("127.0.0.1:9001")).unwrap(); } + #[test] + fn stale_connection_cleanup_cannot_remove_a_replacement() { + let addr = "127.0.0.1:9001"; + let current = Arc::new(ConnectionInstance::new()); + let stale = Arc::new(ConnectionInstance::new()); + let mut peers = HashMap::from([( + addr.to_string(), + PeerConnection { + info: PeerInfo::new(addr), + connection_info: None, + instance: Arc::clone(¤t), + state: PeerState::Connected, + serialization_format: SerializationFormat::Bincode, + writer: None, + _slot: test_slot(), + }, + )]); + + assert!(remove_connection_if_current(&mut peers, addr, &stale).is_none()); + assert!(peers.contains_key(addr)); + assert!(remove_connection_if_current(&mut peers, addr, ¤t).is_some()); + assert!(!peers.contains_key(addr)); + } + #[tokio::test] async fn mark_peer_failed_returns_updated_connected_count() { let node = Node::new(NodeConfig::new(NodeId::new("test"), "127.0.0.1:9000")); @@ -1223,6 +2148,8 @@ mod tests { "127.0.0.1:9001".to_string(), PeerConnection { info: PeerInfo::new("127.0.0.1:9001").with_node_id(NodeId::new("peer-a")), + connection_info: None, + instance: Arc::new(ConnectionInstance::new()), state: PeerState::Connected, serialization_format: SerializationFormat::Bincode, writer: None, @@ -1233,6 +2160,8 @@ mod tests { "127.0.0.1:9002".to_string(), PeerConnection { info: PeerInfo::new("127.0.0.1:9002").with_node_id(NodeId::new("peer-b")), + connection_info: None, + instance: Arc::new(ConnectionInstance::new()), state: PeerState::Connected, serialization_format: SerializationFormat::Bincode, writer: None, @@ -1241,40 +2170,301 @@ mod tests { ); } - let (node_id, connected) = node.mark_peer_failed("127.0.0.1:9001").await.unwrap(); + let instance = Arc::clone(&node.peers.read().await["127.0.0.1:9001"].instance); + let (node_id, connected) = node + .mark_peer_failed("127.0.0.1:9001", &instance) + .await + .unwrap(); assert_eq!(node_id, NodeId::new("peer-a")); assert_eq!(connected, 1); + assert!( + node.mark_peer_failed("127.0.0.1:9001", &instance) + .await + .is_none() + ); + } + + #[tokio::test] + async fn registry_prunes_finished_tasks_before_admission() { + let tasks = StdMutex::new(TaskRegistry::default()); + let (done_tx, done_rx) = tokio::sync::oneshot::channel(); + spawn_task(&tasks, async move { + done_tx.send(()).unwrap(); + }) + .unwrap(); + done_rx.await.unwrap(); + spawn_task(&tasks, std::future::pending()).unwrap(); + assert_eq!(tasks.lock().unwrap().tasks.len(), 1); + ShutdownTasks::close(&tasks).join(Duration::ZERO).await; + } + + #[tokio::test] + async fn registry_rejects_late_admission_and_drops_the_unspawned_future() { + let tasks = StdMutex::new(TaskRegistry::default()); + let mut shutdown = ShutdownTasks::close(&tasks); + let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel::<()>(); + assert!( + spawn_task(&tasks, async move { + let _sender = dropped_tx; + panic!("late task must never run"); + }) + .is_err() + ); + assert!(dropped_rx.await.is_err()); + shutdown.join(Duration::ZERO).await; + assert!(shutdown.tasks.is_empty()); + } + + #[tokio::test] + async fn concurrent_registration_is_joined_or_rejected() { + let node = Node::new(NodeConfig::new(NodeId::new("test"), "127.0.0.1:0")); + let registry = Arc::clone(&node.tasks); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let (admitted_tx, admitted_rx) = tokio::sync::oneshot::channel(); + spawn_task(&node.tasks, async move { + entered_tx.send(()).unwrap(); + release_rx.await.unwrap(); + let admitted = spawn_task(®istry, async {}); + admitted_tx.send(admitted.is_ok()).unwrap(); + }) + .unwrap(); + entered_rx.await.unwrap(); + let mut shutdown = tokio_test::task::spawn(node.shutdown()); + assert!(shutdown.poll().is_pending()); + release_tx.send(()).unwrap(); + assert!(!admitted_rx.await.unwrap()); + shutdown.await; + let registry = node.tasks.lock().unwrap(); + assert!(registry.closed); + assert!(registry.tasks.is_empty()); + } + + #[tokio::test] + async fn cancelled_shutdown_aborts_tasks_and_retains_handles_for_retry() { + let node = Node::new(NodeConfig::new(NodeId::new("test"), "127.0.0.1:0")); + let (dropped_tx, dropped_rx) = tokio::sync::oneshot::channel::<()>(); + spawn_task(&node.tasks, async move { + let _sender = dropped_tx; + std::future::pending::<()>().await; + }) + .unwrap(); + let mut shutdown = tokio_test::task::spawn(node.shutdown()); + assert!(shutdown.poll().is_pending()); + drop(shutdown); + assert!( + dropped_rx.await.is_err(), + "cancelled shutdown detached a task" + ); + assert_eq!(node.tasks.lock().unwrap().tasks.len(), 1); + assert!(node.start_listener().await.is_err()); + node.shutdown().await; + assert!(node.tasks.lock().unwrap().tasks.is_empty()); + assert_eq!(node.connected_peer_count().await, 0); } #[tokio::test] - async fn track_task_prunes_finished_handles_before_push() { - let tasks = Arc::new(Mutex::new(Vec::new())); - let finished = tokio::spawn(async {}); + async fn outbound_handshake_finishing_after_shutdown_cannot_register_a_peer() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + let node = Arc::new(Node::new(NodeConfig::new( + NodeId::new("test"), + "127.0.0.1:0", + ))); + let client = Arc::clone(&node); + let connect = tokio::spawn(async move { client.connect_to_peer(&addr).await }); + let (mut stream, _) = listener.accept().await.unwrap(); + read_message( + &mut stream, + DEFAULT_MAX_MESSAGE_SIZE, + Duration::from_secs(1), + ) + .await + .unwrap(); + node.shutdown().await; + write_message( + &mut stream, + &Message::hello_ack_with_format(NodeId::new("peer"), SerializationFormat::Bincode), + SerializationFormat::Bincode, + Duration::from_secs(1), + ) + .await + .unwrap(); + assert!(connect.await.unwrap().is_err()); + assert!(node.connected_peers().await.is_empty()); + assert!(node.tasks.lock().unwrap().tasks.is_empty()); + assert_eq!(node.connection_slots.available_permits(), DEFAULT_MAX_PEERS); + } + #[tokio::test] + async fn cancelling_connect_during_event_backpressure_keeps_read_task_owned() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + let mut node = Node::new( + NodeConfig::new(NodeId::new("test"), "127.0.0.1:0").with_event_channel_capacity(1), + ); + let mut events = node.take_event_receiver().unwrap(); + node.event_tx + .try_send(NodeEvent::OpsReceived { + from: NodeId::new("queued"), + ops: vec![], + }) + .unwrap(); + let node = Arc::new(node); + let client = Arc::clone(&node); + let endpoint = addr.clone(); + let connect = tokio::spawn(async move { client.connect_to_peer(&endpoint).await }); + let (mut stream, _) = listener.accept().await.unwrap(); + read_message( + &mut stream, + DEFAULT_MAX_MESSAGE_SIZE, + Duration::from_secs(1), + ) + .await + .unwrap(); + write_message( + &mut stream, + &Message::hello_ack_with_format(NodeId::new("peer"), SerializationFormat::Bincode), + SerializationFormat::Bincode, + Duration::from_secs(1), + ) + .await + .unwrap(); timeout(Duration::from_secs(1), async { - loop { - if finished.is_finished() { - break; - } + while !node.is_connected_addr(&addr).await { tokio::task::yield_now().await; } }) .await .unwrap(); + assert!(!connect.is_finished()); + connect.abort(); + assert!(connect.await.unwrap_err().is_cancelled()); + assert_eq!(node.tasks.lock().unwrap().tasks.len(), 1); + assert!(matches!( + events.recv().await, + Some(NodeEvent::OpsReceived { .. }) + )); + assert!(matches!( + events.recv().await, + Some(NodeEvent::PeerConnected { .. }) + )); + node.shutdown().await; + assert!(node.tasks.lock().unwrap().tasks.is_empty()); + assert!(node.connected_peers().await.is_empty()); + } - track_task(&tasks, finished).await; - assert_eq!(tasks.lock().await.len(), 1); + #[tokio::test(start_paused = true)] + async fn shutdown_uses_one_total_grace_for_all_tasks() { + let node = Node::new(NodeConfig::new(NodeId::new("test"), "127.0.0.1:0")); + let mut dropped = Vec::new(); + for _ in 0..8 { + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + dropped.push(rx); + spawn_task(&node.tasks, async move { + let _sender = tx; + std::future::pending::<()>().await; + }) + .unwrap(); + } + let started = tokio::time::Instant::now(); + node.shutdown().await; + assert_eq!(started.elapsed(), TASK_SHUTDOWN_GRACE); + for task in dropped { + assert!(task.await.is_err()); + } + assert!(node.tasks.lock().unwrap().tasks.is_empty()); + } - let pending = tokio::spawn(async { - tokio::time::sleep(Duration::from_secs(60)).await; - }); - track_task(&tasks, pending).await; + #[tokio::test] + async fn dropping_node_aborts_owned_tasks_even_when_registry_is_shared() { + let node = Node::new(NodeConfig::new(NodeId::new("test"), "127.0.0.1:0")); + let registry = Arc::clone(&node.tasks); + let (tx, rx) = tokio::sync::oneshot::channel::<()>(); + let task_registry = Arc::clone(®istry); + spawn_task(®istry, async move { + let _registry = task_registry; + let _sender = tx; + std::future::pending::<()>().await; + }) + .unwrap(); + drop(node); + assert!(rx.await.is_err()); + assert!(registry.lock().unwrap().closed); + ShutdownTasks::close(®istry).join(Duration::ZERO).await; + } - let mut tasks = tasks.lock().await; - assert_eq!(tasks.len(), 1); - for task in tasks.drain(..) { - task.abort(); + #[tokio::test] + async fn stale_writer_failure_does_not_fail_or_emit_disconnect_for_replacement() { + for broadcast in [false, true] { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let stream = TcpStream::connect(listener.local_addr().unwrap()) + .await + .unwrap(); + let (_remote, _) = listener.accept().await.unwrap(); + let (_reader, writer) = tokio::io::split(NetStream::Plain(stream)); + let writer = Arc::new(Mutex::new(writer)); + let mut held_writer = writer.lock().await; + held_writer.shutdown().await.unwrap(); + let mut node = Node::new(NodeConfig::new(NodeId::new("test"), "127.0.0.1:0")); + let mut events = node.take_event_receiver().unwrap(); + let addr = "127.0.0.1:9001"; + node.peers.write().await.insert( + addr.into(), + PeerConnection { + info: PeerInfo::new(addr).with_node_id(NodeId::new("old")), + connection_info: None, + instance: Arc::new(ConnectionInstance::new()), + state: PeerState::Connected, + serialization_format: SerializationFormat::Bincode, + writer: Some(Arc::clone(&writer)), + _slot: test_slot(), + }, + ); + let mut send = tokio_test::task::spawn(async { + if broadcast { + node.broadcast_message(Message::ping()).await + } else { + node.send_message_to_addr(addr, Message::ping()).await + } + }); + // The snapshot has been taken, but writing is blocked on our writer lock. + assert!(send.poll().is_pending()); + let stale = Arc::clone(&node.peers.read().await[addr].instance); + let replacement = Arc::new(ConnectionInstance::new()); + node.peers.write().await.insert( + addr.into(), + PeerConnection { + info: PeerInfo::new(addr).with_node_id(NodeId::new("replacement")), + connection_info: None, + instance: Arc::clone(&replacement), + state: PeerState::Connected, + serialization_format: SerializationFormat::Bincode, + writer: None, + _slot: test_slot(), + }, + ); + // Cancellation must release the snapshot without acquiring this lock. + assert!( + timeout(Duration::from_secs(1), send) + .await + .unwrap() + .is_err() + ); + assert!(node.mark_peer_failed(addr, &stale).await.is_none()); + drop(held_writer); + assert!(node.is_connected_addr(addr).await); + assert_eq!( + node.connected_peers().await, + [(addr.into(), NodeId::new("replacement"))] + ); + assert!(events.try_recv().is_err()); + assert!(Arc::ptr_eq( + &node.peers.read().await[addr].instance, + &replacement + )); + node.shutdown().await; } } @@ -1287,6 +2477,8 @@ mod tests { "127.0.0.1:9001".to_string(), PeerConnection { info: PeerInfo::new("127.0.0.1:9001").with_node_id(NodeId::new("peer-a")), + connection_info: None, + instance: Arc::new(ConnectionInstance::new()), state: PeerState::Connected, serialization_format: SerializationFormat::Bincode, writer: None, @@ -1297,6 +2489,8 @@ mod tests { "127.0.0.1:9002".to_string(), PeerConnection { info: PeerInfo::new("127.0.0.1:9002").with_node_id(NodeId::new("peer-b")), + connection_info: None, + instance: Arc::new(ConnectionInstance::new()), state: PeerState::Failed, serialization_format: SerializationFormat::Bincode, writer: None, @@ -1310,6 +2504,38 @@ mod tests { assert!(!node.is_connected_addr("127.0.0.1:9003").await); } + #[tokio::test] + async fn connected_peers_is_sorted_and_excludes_failed_connections() { + let node = Node::new(NodeConfig::new(NodeId::new("test"), "127.0.0.1:0")); + for (addr, state) in [ + ("z.example:9000", PeerState::Connected), + ("a.example:9000", PeerState::Connected), + ("failed.example:9000", PeerState::Failed), + ] { + node.peers.write().await.insert( + addr.into(), + PeerConnection { + info: PeerInfo::new(addr).with_node_id(NodeId::new(addr)), + connection_info: None, + instance: Arc::new(ConnectionInstance::new()), + state, + serialization_format: SerializationFormat::Bincode, + writer: None, + _slot: test_slot(), + }, + ); + } + assert_eq!( + node.connected_peers().await, + [ + ("a.example:9000".into(), NodeId::new("a.example:9000")), + ("z.example:9000".into(), NodeId::new("z.example:9000")), + ] + ); + node.shutdown().await; + assert!(node.connected_peers().await.is_empty()); + } + #[test] fn node_config_allows_custom_wire_limits() { let config = NodeConfig::new(NodeId::new("test"), "127.0.0.1:9000") @@ -1658,10 +2884,70 @@ mod tests { assert_eq!(peer.serialization_format, SerializationFormat::Json); drop(peers); + let connection = node_a + .connection_info(&addr_b.to_string()) + .await + .expect("active connection metadata"); + assert_eq!(connection.transport_addr, addr_b.to_string()); + assert_eq!(connection.dialed_endpoint, Some(addr_b.to_string())); + assert_eq!(connection.direction, ConnectionDirection::Outbound); + assert_eq!(connection.identity.node_id, NodeId::new("node-b")); + assert_eq!( + connection.identity.verification, + PeerIdentityVerification::Unverified + ); node_a.shutdown().await; node_b.shutdown().await; } + #[tokio::test] + async fn self_node_id_is_rejected_after_handshake() { + let node = Node::new( + NodeConfig::new(NodeId::new("same-node"), "127.0.0.1:0") + .with_socket_timeout(Duration::from_secs(1)), + ); + let addr = node.start_listener().await.unwrap(); + + assert!(node.connect_to_peer(&addr.to_string()).await.is_err()); + assert_eq!(node.connected_peer_count().await, 0); + + node.shutdown().await; + } + + #[tokio::test] + async fn duplicate_outbound_attempt_is_rejected_while_handshake_is_pending() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap().to_string(); + let (accepted_tx, accepted_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = tokio::sync::oneshot::channel(); + let server = tokio::spawn(async move { + let (stream, _) = listener.accept().await.unwrap(); + let _ = accepted_tx.send(()); + let _ = release_rx.await; + drop(stream); + }); + let node = Arc::new(Node::new( + NodeConfig::new(NodeId::new("client"), "127.0.0.1:0") + .with_socket_timeout(Duration::from_secs(2)), + )); + let first_node = Arc::clone(&node); + let first_addr = addr.clone(); + let first = tokio::spawn(async move { first_node.connect_to_peer(&first_addr).await }); + accepted_rx.await.unwrap(); + + assert!(matches!( + node.connect_to_peer(&addr).await, + Err(NetError::ConnectionFailed(message)) + if message.contains("connection attempt already in progress") + && message.contains(&addr) + )); + + let _ = release_tx.send(()); + assert!(first.await.unwrap().is_err()); + server.await.unwrap(); + node.shutdown().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "stress test: opens 1000 local TCP connections"] async fn thousand_simultaneous_connections_do_not_crash() { diff --git a/crates/nx-net/src/peer.rs b/crates/nx-net/src/peer.rs index 6ec7f4f..83fa8cb 100644 --- a/crates/nx-net/src/peer.rs +++ b/crates/nx-net/src/peer.rs @@ -5,6 +5,41 @@ use std::net::SocketAddr; /// Identifier of a peer (based on NodeId). pub type PeerId = NodeId; +/// Direction in which an active transport connection was established. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ConnectionDirection { + Inbound, + Outbound, +} + +/// Evidence binding the handshake NodeId to the transport peer. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum PeerIdentityVerification { + /// The NodeId was derived from and matched against the TLS certificate. + CertificateBound, + /// The transport did not cryptographically bind the claimed NodeId. + Unverified, +} + +/// Identity learned during the wire handshake and how it was verified. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PeerIdentity { + pub node_id: NodeId, + pub verification: PeerIdentityVerification, +} + +/// Immutable facts about one active connection. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PeerConnectionInfo { + /// Actual remote TCP endpoint. + pub transport_addr: String, + /// Discovery/configuration endpoint used to dial, absent for inbound peers. + #[serde(skip_serializing_if = "Option::is_none")] + pub dialed_endpoint: Option, + pub direction: ConnectionDirection, + pub identity: PeerIdentity, +} + #[allow(dead_code)] /// Connection state of a peer. #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/nx-net/tests/api_compat_v014.rs b/crates/nx-net/tests/api_compat_v014.rs new file mode 100644 index 0000000..0b8dc31 --- /dev/null +++ b/crates/nx-net/tests/api_compat_v014.rs @@ -0,0 +1,73 @@ +use std::time::Duration; + +use nx_net::{Message, MessageKind, NetError, NodeConfig, SerializationFormat, WireError}; +use nx_sync::NodeId; + +fn match_message_kind(kind: &MessageKind) { + match kind { + MessageKind::Hello { .. } + | MessageKind::HelloAck { .. } + | MessageKind::PushOps { .. } + | MessageKind::PushOpsAck { .. } + | MessageKind::PullSince { .. } + | MessageKind::Ping + | MessageKind::Pong + | MessageKind::Error { .. } => {} + } +} + +fn match_wire_error(error: &WireError) { + match error { + WireError::ProtocolMismatch { .. } + | WireError::OpRejected { .. } + | WireError::RateLimited { .. } + | WireError::NotAuthorized { .. } + | WireError::Internal { .. } => {} + } +} + +fn match_net_error(error: &NetError) { + match error { + NetError::Io(_) + | NetError::Serialization(_) + | NetError::BinarySerialization(_) + | NetError::BinaryDeserialization(_) + | NetError::ConnectionFailed(_) + | NetError::PeerDisconnected(_) + | NetError::InvalidMessage(_) + | NetError::Wire(_) + | NetError::MessageTooLarge { .. } + | NetError::Timeout + | NetError::ChannelClosed + | NetError::TlsError(_) + | NetError::PeerNotAllowed(_) + | NetError::PeerLimitReached(_) + | NetError::NodeIdMismatch { .. } => {} + } +} + +#[test] +fn public_v014_surface_remains_source_compatible() { + let config = NodeConfig { + node_id: NodeId::new("compat"), + listen_addr: "127.0.0.1:0".into(), + initial_peers: Vec::new(), + tls: None, + max_peers: 8, + max_message_size: 1024, + socket_timeout: Duration::from_secs(1), + serialization_format: SerializationFormat::Bincode, + event_channel_capacity: 8, + }; + let _node = nx_net::Node::new(config); + + let message = Message::ping(); + match_message_kind(&message.kind); + match_wire_error(&WireError::Internal { + reason: "compat".into(), + }); + match_net_error(&NetError::Timeout); + + message.to_bytes().unwrap(); + message.to_json_bytes().unwrap(); +} diff --git a/crates/nx-sdk/Cargo.toml b/crates/nx-sdk/Cargo.toml index 5b3925a..ec027f0 100644 --- a/crates/nx-sdk/Cargo.toml +++ b/crates/nx-sdk/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" edition = "2024" license.workspace = true diff --git a/crates/nx-sdk/src/db.rs b/crates/nx-sdk/src/db.rs index 97ebd84..ca3b122 100644 --- a/crates/nx-sdk/src/db.rs +++ b/crates/nx-sdk/src/db.rs @@ -206,7 +206,7 @@ pub fn scan_page_after( } } -/// keys_page(prefix, cursor, limit) -> Result, NxError> +/// `keys_page(prefix, cursor, limit) -> Result, NxError>` #[must_use = "this SDK call can fail; handle the Result"] pub fn keys_page(prefix: &str, cursor: u64, limit: u32) -> Result>> { let mut cap: usize = 256; @@ -243,7 +243,7 @@ pub fn keys_page(prefix: &str, cursor: u64, limit: u32) -> Result>> } } -/// keys_page_after(prefix, start_after_key, limit) -> Result, NxError> +/// `keys_page_after(prefix, start_after_key, limit) -> Result, NxError>` #[must_use = "this SDK call can fail; handle the Result"] pub fn keys_page_after( prefix: &str, @@ -286,7 +286,7 @@ pub fn keys_page_after( } } -/// scan(prefix) -> Result, NxError> +/// `scan(prefix) -> Result, NxError>` #[must_use = "this SDK call can fail; handle the Result"] pub fn scan(prefix: &str) -> Result, Vec)>> { let mut last_key: Option> = None; @@ -308,7 +308,7 @@ pub fn scan(prefix: &str) -> Result, Vec)>> { } } -/// keys(prefix) -> Result, NxError> +/// `keys(prefix) -> Result, NxError>` #[must_use = "this SDK call can fail; handle the Result"] pub fn keys(prefix: &str) -> Result>> { let mut last_key: Option> = None; diff --git a/crates/nx-store/Cargo.toml b/crates/nx-store/Cargo.toml index 11924fa..e42c589 100644 --- a/crates/nx-store/Cargo.toml +++ b/crates/nx-store/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nx-store" -version = "0.1.4" +version = "0.1.5" edition = "2024" license.workspace = true diff --git a/crates/nx-sync/Cargo.toml b/crates/nx-sync/Cargo.toml index 71aa151..ffabfdd 100644 --- a/crates/nx-sync/Cargo.toml +++ b/crates/nx-sync/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "nx-sync" -version = "0.1.4" +version = "0.1.5" edition = "2024" license.workspace = true diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index 058cc1a..5640f7d 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -3,7 +3,7 @@ jsonSchemaDialect: https://json-schema.org/draft/2020-12/schema info: title: Numax Management API - version: 0.1.4 + version: 0.1.5 description: | Authenticated API for operating a single Numax node. diff --git a/docs/nx-site/package-lock.json b/docs/nx-site/package-lock.json index 7abaf9f..59a98f4 100644 --- a/docs/nx-site/package-lock.json +++ b/docs/nx-site/package-lock.json @@ -1,12 +1,12 @@ { "name": "nx-site", - "version": "0.1.4", + "version": "0.1.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "nx-site", - "version": "0.1.4", + "version": "0.1.5", "dependencies": { "@astrojs/starlight": "^0.32.0", "astro": "^5.0.0" diff --git a/docs/nx-site/package.json b/docs/nx-site/package.json index 27db0b8..97e0e89 100644 --- a/docs/nx-site/package.json +++ b/docs/nx-site/package.json @@ -1,6 +1,6 @@ { "name": "nx-site", - "version": "0.1.4", + "version": "0.1.5", "private": true, "type": "module", "scripts": { diff --git a/docs/nx-site/src/content/docs/concepts/gossip-protocol.md b/docs/nx-site/src/content/docs/concepts/gossip-protocol.md index c560a39..7cb30f1 100644 --- a/docs/nx-site/src/content/docs/concepts/gossip-protocol.md +++ b/docs/nx-site/src/content/docs/concepts/gossip-protocol.md @@ -5,7 +5,10 @@ description: How Numax moves operations between peers. This page explains what gossip means in Numax, what the current sync layer already does, and what will arrive in the peer-discovery releases. -The short version: **today Numax uses configured peers, direct broadcasts and periodic anti-entropy.** Future releases will turn that into dynamic peer discovery with SWIM-style membership and K-fanout gossip. +The short version: **Numax discovers connection candidates, broadcasts directly +to active peers, and repairs missed operations through periodic anti-entropy.** +Discovery is dynamic in `v0.1.5`; SWIM-style membership and K-fanout data gossip +remain future work. --- @@ -31,9 +34,19 @@ Each operation has a globally unique `OpId`, the node that produced it, and the ## What exists today -The current implementation is intentionally simple and deterministic. +The current data-replication implementation remains intentionally simple and +deterministic. A node obtains endpoint candidates from static, bootstrap, mDNS, +DNS-SRV or file providers. The same bounded, updateable candidate snapshot feeds +initial dialing and reconnect. Anti-entropy uses all active connections, +including inbound peers and connections whose candidates have been removed. +Starting with no candidates is valid; later provider updates wake the +connection machinery. Local readiness does not imply peer convergence. -Numax does not yet have dynamic peer discovery. A node knows the peers configured at startup or added explicitly through the runtime API. When an operation is produced locally, the sync manager queues it and sends it to the currently connected peers. +Candidates are not members or peers yet. A candidate becomes an active peer +only after connection admission, the normal wire handshake, TLS identity +binding when configured, and allowlist authorization. When an operation is +produced locally, the sync manager queues it and sends it to the currently +connected peers. ``` local CRDT host call @@ -68,8 +81,13 @@ Peer communication is handled by `nx-net`. The current wire protocol defines the | `PullSince` | Ask a peer for retained operations. Today this is usually sent with `None`. | | `Ping` / `Pong` | Keepalive message types. A received `Ping` is answered with `Pong`. | | `Error` | Structured wire error sent before rejecting a request or closing a connection. | +| `BootstrapHello` | Start a one-shot authenticated bootstrap request with cluster, advertisement and result limit. | +| `BootstrapAck` | Return the seed identity, matching cluster, negotiated format, bounded endpoint suggestions and their lease. | -The protocol version is currently `4`. Peers negotiate either `Bincode` or `Json`, with `Bincode` as the production default and `Json` available for debug-style interoperability. +The protocol version is currently `5`. Peers negotiate either `Bincode` or +`Json`, with `Bincode` as the production default and `Json` available for +debug-style interoperability. Version `5` is deliberately incompatible with +the version `4` wire contract from Numax `v0.1.4`. --- @@ -104,6 +122,35 @@ Serialization-format negotiation does not override protocol compatibility. The rules for evolving this contract are defined in [Wire Versioning](/numax/design/wire-versioning/). +### Bootstrap handshake + +Bootstrap uses the same listener but a separate, one-shot first message: + +```text +client -> seed: BootstrapHello( + node_id, protocol_version, supported_formats, preferred_format, + cluster_id, advertised_endpoint?, max_results +) +seed -> client: BootstrapAck( + node_id, protocol_version, selected_format, + cluster_id, candidates, candidate_ttl_ms +) +connection closes +``` + +The seed authenticates the requesting node using the same TLS certificate +binding and allowlist checks as a normal peer handshake, validates cluster and +advertised endpoint, then records that endpoint under a bounded lease. The +client likewise authenticates the seed and validates the complete response. +Cluster mismatch or an invalid request yields `BootstrapRejected`. + +The one-shot exchange never enters the active peer map and emits no +`PeerConnected` event. Authentication proves only who answered and who made the +request; it does not vouch for any endpoint in `candidates`. Each suggestion is +fed into normal reconnection and must authenticate independently before CRDT +traffic can flow. Response count, cache size, candidate TTL, message size, +socket time and concurrent client queries are all bounded. + --- ## Broadcast path @@ -130,7 +177,10 @@ If a peer is disconnected, it does not receive the immediate push. That is why a Anti-entropy is the repair loop. -Every `anti_entropy_interval` seconds, a node asks each connected configured peer for retained operations using `PullSince`. +Every `anti_entropy_interval`, a node asks each active connection for retained +operations using `PullSince`. This cadence is independent of discovery churn; +missed ticks are skipped rather than replayed in a burst. Candidate removal +stops future reconnect attempts, not repair over an already admitted connection. Today the request is conservative: it asks for the bounded op-log rather than relying on a single "last seen op id" as a causal frontier. That matters because one newer operation does not prove that every older operation arrived. @@ -152,17 +202,19 @@ node B returns retained ops node A applies only unseen OpIds ``` -The op-log is bounded, so anti-entropy is a practical catch-up mechanism, not an infinite historical archive. +The op-log and deduplication history are bounded, so anti-entropy is a practical +catch-up mechanism, not an infinite historical archive or state transfer. +Rediscovery alone cannot guarantee recovery when the required history is gone. --- ## Peer health and reconnect -Configured peers have a small health state: +Current candidates have a small health state: | State | Meaning | |---|---| -| `Healthy` | The configured peer is connected or recently connected successfully. | +| `Healthy` | The candidate is connected or recently connected successfully. | | `Suspect` | A connection attempt failed, but the peer has not crossed the failure threshold. | | `Dead` | Consecutive failures reached `peer_dead_after_failures`. | @@ -175,7 +227,8 @@ Reconnect uses exponential backoff: | Dead after failures | `3` | | Anti-entropy interval | `30s` | -This is simple failure tracking for configured peers. It is not a full membership protocol yet. +This is simple failure tracking for discovery candidates, including configured +peers. It is not a full membership protocol yet. --- @@ -183,7 +236,6 @@ This is simple failure tracking for configured peers. It is not a full membershi The current release line does **not** yet provide: -- automatic peer discovery, - SWIM membership, - Lifeguard-style failure detection, - phi-accrual failure detection, @@ -192,28 +244,33 @@ The current release line does **not** yet provide: - NAT traversal, - causal frontier metadata for precise incremental pulls. -If you see "gossip" in the current docs, read it as the sync layer that propagates and repairs CRDT operations between known peers. The more formal gossip protocol is planned in the peer-discovery work. +If you see "gossip" in the current docs, distinguish bootstrap gossip β€” a +bounded exchange of endpoint suggestions β€” from data gossip. Current CRDT +propagation is still a broadcast to all active peers, with anti-entropy as its +repair path. Bootstrap suggestions are not membership state. --- -## What comes next +## Current foundations and next steps -Peer discovery is planned in two steps. +Peer discovery foundations are implemented in the current `v0.1.5` release; +membership and K-fanout remain planned for `v0.1.6`. ### v0.1.5 - Peer Discovery: Foundations -This release introduces the discovery abstraction and the first discovery backends. - -Planned work: - -- `PeerDiscovery` trait with `discover()`, `announce()` and `watch()`. -- `StaticDiscovery`, preserving the current configured-peer behavior. -- Bootstrap discovery: join through one known address and learn other peers. -- mDNS discovery for LAN/dev setups. -- DNS-SRV discovery for environments that already publish service records. -- File-watch discovery for orchestrators and Kubernetes-style setups. +This release introduces the `PeerDiscovery` contract and five Rust provider +implementations: static configuration, authenticated bootstrap, LAN mDNS, +DNS-SRV and an externally updated peer file. Snapshot/watch handoff is atomic, +delivery is bounded with explicit overflow, and provider tasks are owned and +stopped by runtime shutdown. -The goal is to stop making every node list every other node manually. +All five modes (`static`, `bootstrap`, `mdns`, `dns-srv`, `file`) are selectable +through `--discovery-mode`, `NX_DISCOVERY_MODE` and the `[discovery]` TOML section, +with precedence CLI > environment > TOML > defaults. Explicit `--peer` entries +continue to contribute a static source alongside the selected dynamic provider; +they are not reinterpreted as bootstrap seeds. Embedders can also compose the +public providers through the `nx-core` Rust API. The detailed semantics are in the +[Peer Discovery Contract](/numax/design/discovery-contract/). ### v0.1.6 - Peer Discovery: SWIM & Gossip K-fanout @@ -247,7 +304,10 @@ Gossip is the fast path. It spreads new operations quickly. Anti-entropy is the repair path. It catches up nodes that were offline, partitioned, slow, or unlucky. -Numax needs both because local-first systems must tolerate temporary disconnection. CRDTs make the merge safe. Gossip moves operations quickly. Anti-entropy makes missed operations recoverable. +Numax needs both because local-first systems must tolerate temporary +disconnection. CRDTs define convergence semantics; dissemination moves +operations between peers, and anti-entropy repairs missed operations while the +required operation and deduplication history remains available. --- diff --git a/docs/nx-site/src/content/docs/design/discovery-contract.md b/docs/nx-site/src/content/docs/design/discovery-contract.md new file mode 100644 index 0000000..0b166f7 --- /dev/null +++ b/docs/nx-site/src/content/docs/design/discovery-contract.md @@ -0,0 +1,449 @@ +--- +title: Peer Discovery Contract +description: Snapshot, event delivery, cancellation, and compatibility guarantees for peer discovery providers. +--- + +## Scope and ownership + +This contract describes peer discovery in `v0.1.5`, the current Numax version. + +The peer discovery abstraction belongs to `nx-core`. It supplies peer endpoint +candidates to runtime orchestration without moving connection management, +authentication, or wire-protocol concerns into discovery providers. + +`PeerDiscovery` defines three operations: + +- `discover()` returns the provider's current snapshot; +- `watch()` subscribes to changes after that snapshot; +- `announce()` asks a provider to publish the local endpoint when it supports + announcements. + +`DiscoveryProvider` gives every source a stable source ID and an optional +candidate lease. `DiscoveryRuntimeConfig` defines the local cluster scope, the +optional advertised endpoint, and the global candidate bound. The coordinator +owns every watch and is the only component that turns provider contributions +into the effective candidate snapshot. + +## Snapshot and watch consistency + +Creating a watch and reading the snapshot bundled with it are one atomic +observation. An update cannot occur between those actions without being +represented either in that snapshot or by a subsequent event. Consumers that +need updates therefore start with `DiscoveryWatch::snapshot()` and then process +the same watch's event stream. The separate `discover()` method is for +point-in-time reads and must not be combined with a later `watch()` call. + +`StaticDiscovery` is immutable. Its provider snapshot preserves the configured +peer list exactly, including input order and duplicate entries, and its watch +produces no change events. The coordinator canonicalizes endpoints and keeps +the first occurrence order, so duplicate configuration entries still result in +only one effective connection candidate. Invalid legacy `--peer` values are +logged and skipped instead of making discovery startup fail. + +Dynamic providers keep a complete ordered view and publish bounded, +revisioned `Observed(DiscoverySnapshot)` events with per-endpoint observation +timestamps. A replacement changes the provider contribution +atomically, including its ordering: consumers never observe a synthetic empty +view between removals and additions. `Replaced`, `Added` and `Removed` remain +available for providers without observation metadata. Providers deduplicate their own snapshots where their +source naturally can repeat endpoints; the coordinator also deduplicates +across providers. Ordering is deterministic for a given set of provider +observations, but it is not a membership or authorization guarantee. + +## Candidate ownership, expiry, and removal + +An effective candidate can have contributions from multiple discovery sources. +`Added` refreshes that source's optional lease. `Removed` deletes only that +source's contribution; the candidate disappears only after its last source is +removed or expires. A leased source survives a watch failure until its lease +expires, while an unleased source is removed when its watch becomes unavailable. +A successful resubscription atomically replaces that source from the new watch +snapshot. + +Freshness is based on successful endpoint observation, not cache publication or +watch subscription time. `Observed` preserves those timestamps in both events +and resubscription snapshots. A successful refresh of an unchanged endpoint +list advances freshness; replaying a cached last-good view after an error does +not renew its lease. Aggregated bootstrap seed and mDNS instance views preserve +each endpoint's observation time rather than refreshing unrelated entries. + +The resulting bounded snapshot drives initial dialing and automatic +reconnection in candidate order. An empty startup snapshot is valid, and the +loops remain alive for later additions. `SyncManager::start()` returns after +local services and their owned background loops are ready; it does not await +peer convergence or successful dialing of every candidate. A stalled initial +handshake therefore does not delay local readiness by one timeout per peer. + +Removing a candidate stops new reconnect attempts; it does not terminate an +already active, admitted connection. Once that connection closes it is not +re-established unless a source adds the endpoint again. Anti-entropy instead +uses all active connection send-address keys, including inbound connections +and peers no longer present in discovery. Its periodic cadence is independent +of candidate churn, and missed ticks are skipped rather than replayed in a +burst. Removal from discovery therefore does not disable repair over a live +connection. + +Anti-entropy pulls the bounded operation log and relies on receiver +deduplication. It is not state transfer and does not guarantee unrestricted +lossless recovery after a partition or restart: required operations and +deduplication history must still be retained. Rediscovery alone does not prove +that a missing-history gap can be repaired. + +## Bounded event delivery + +The public Rust constant `MAX_DISCOVERY_EVENT_CAPACITY` is 4096; the default +`DEFAULT_DISCOVERY_EVENT_CAPACITY` remains 128. Both are exported from +`nx_core::discovery` and the `nx_core` crate root. All four dynamic provider +constructors validate `event_capacity` in `1..=4096` before channel/state +allocation, spawning or provider I/O, returning +`DiscoveryError::InvalidConfiguration` outside that range. The bound is on +event slots, not on total process memory or the aggregate candidate snapshot. +Tokio broadcast channels may round the requested capacity up to a power of +two; the common maximum still bounds that rounded capacity. mDNS also uses +the validated capacity for its bounded announcement-request channel. + +Watch delivery is bounded. A provider must not grow an unbounded queue when a +consumer is slow. If changes exceed the available capacity, overflow is exposed +to the consumer as an explicit provider error rather than silently dropping +events. Revisions are contiguous and strictly increasing after the watch +snapshot; a discontinuity is also an explicit error. After either condition, +incremental state is no longer authoritative and the consumer must create a +new watch and use its bundled snapshot before continuing. + +Dropping a watch cancels that subscription. Provider closure terminates the +watch. `StaticDiscovery` owns no background task, so dropping it or its watch +requires no asynchronous shutdown or task join. + +## Announcements and errors + +Announcement support is a provider capability. `StaticDiscovery::announce()` +returns the explicit unsupported-operation error; it does not silently succeed +and does not alter the configured snapshot. Other provider failures are returned +through the typed discovery error boundary so callers can distinguish an +unsupported capability, closed delivery, and overflow requiring a resnapshot. + +Providers declare announcements unsupported, optional, or required. Required +announcements make startup fail if no dialable local endpoint can be derived. +Provider `shutdown()` owns withdrawal of announcements and termination of any +provider-internal work. The coordinator stops and joins every watch task and +calls every provider shutdown hook during normal shutdown and partial-startup +rollback. Provider operations have a finite timeout so a stuck implementation +cannot keep runtime shutdown alive indefinitely. + +`request_shutdown()` makes a dynamic provider permanently stopped; subsequent +`shutdown()` calls wait for the same owned completion and can report the same +failure. Explicit shutdown is terminal, not a restart request. Dropping a +shutdown waiter does not cancel generation cleanup. Dropping the provider +signals cancellation; the supervisor owns bounded cleanup while the runtime +remains alive. Runtime teardown is not a guarantee of external withdrawal. + +Unexpected worker exit is different from explicit shutdown. The supervisor +joins the worker, clears its stale view, invalidates existing watches and +completes provider-specific cleanup before admitting any replacement generation. +Watch invalidation or a finished worker alone is not a restart barrier. During +finalization, a fresh `discover()` or `watch()` fails rather than subscribing to +the exited producer. After successful cleanup, a later operation may start one +new generation if the worker outcome permits recovery (including a worker +panic or retryable error); a fatal worker error blocks restart. A cleanup error +or panic also blocks restart, even when that cleanup error is marked retryable. +Concurrent subscribers share restart admission rather than starting overlapping +generations. Worker errors and panics remain observable during shutdown, even +when they race a stop request. + +Bootstrap and mDNS retain desired announcement intent across recoverable +unexpected exits, but do not retain stale candidate views. Explicit shutdown +clears that intent, including when requested after unexpected finalization. +Successful cleanup means the provider's local cleanup contract completed; it +does not imply that every remote peer received a withdrawal or goodbye. + +## Provider contracts + +All provider limits are checked before a view is exposed to the coordinator. +The runtime-wide candidate limit remains an additional bound after different +sources are combined. + +### StaticDiscovery + +`StaticDiscovery::new(peers)` is the compatibility adapter for configured +peers. It performs no I/O, never refreshes or expires entries, preserves the +input list byte-for-byte, and does not support announcements. An empty list is +valid. + +`StaticDiscovery::with_event_capacity(peers, capacity)` remains infallible and +clamps capacity to `[1, 4096]`: zero becomes one, and values above the maximum +(including `usize::MAX`) become 4096. It does not truncate or reorder peers or +remove duplicates. `StaticDiscovery::try_with_event_capacity(peers, capacity)` +is the strict alternative: it returns `Result` and rejects +zero or values above `MAX_DISCOVERY_EVENT_CAPACITY` with +`InvalidConfiguration` before channel allocation. Valid inputs preserve the +same peer snapshot semantics. Static discovery has no dynamic worker lifecycle. + +### BootstrapGossipDiscovery + +`BootstrapGossipDiscovery` contacts a bounded, ordered seed list through the +one-shot `BootstrapHello`/`BootstrapAck` exchange. Startup with no responses is +valid: the initial provider snapshot is empty and probing continues in the +background. Seed addresses are canonicalized and deduplicated while retaining +their first configured occurrence. + +Each request optionally advertises the caller's endpoint and asks for at most +the configured number of results. Response capacity is in `1..=4096`, matching +`nx_net::MAX_BOOTSTRAP_RESPONSE_CAPACITY`; bootstrap configuration rejects +larger capacities before querying a seed. This is a bootstrap response bound, +not a universal cap on the aggregate runtime snapshot from all discovery sources. +The bootstrap provider's `max_candidates` must also fit the client's response +capacity. Each retained seed view includes the seed itself first, followed by +deduplicated suggestions, truncated to that provider limit. Views from multiple +seeds are flattened in configured seed order, deduplicated again and capped by +the provider's `max_candidates`. The coordinator applies its separate global +candidate limit after combining sources. Returned +entries expire at the earlier of the seed-provided lease and the provider's +`stale_after` bound. Failed probes retain an unexpired last valid view; expired +views are removed at their deadline even while another seed query is still in +flight. Each successful seed response is published without waiting for the +remaining seeds in the refresh pass. + +Probe failures use exponential retry bounded by `retry_initial` and +`retry_max`; a success restores `refresh_interval`. Fatal wire failures such as +protocol mismatch or bootstrap request rejection disable that seed for the +current worker generation. Bootstrap announcement support is required. A seed +is tracked conservatively for withdrawal before an advertising query is +awaited: the seed may have accepted the endpoint even if the response is lost, +decoding fails, or the query is cancelled. Tracking is therefore not restricted +to acknowledged successful announcements and is bounded by the configured +seed list. + +Cleanup stops and joins the probe loop, then attempts withdrawal from tracked +seeds within a shared four-second budget, dividing the remaining time among +remaining seeds. This is **bounded best effort**, not guaranteed delivery to +every seed: query failures and timeouts are logged, the budget may expire, and +an `Ok(())` cleanup result does not prove remote withdrawal. Local tracking and +the candidate view are cleared even after a worker panic or a cancelled +shutdown waiter. A seed that misses withdrawal can retain the advertisement +until its bounded lease expires. This best-effort bootstrap contract is distinct +from mDNS's checked daemon-acknowledgement cleanup below. + +The seed authenticates the requester before caching its advertisement, and the +client authenticates the responding seed according to the normal TLS and +allowlist policy. That authentication covers only the two participants in the +bootstrap exchange. Every returned endpoint is still an untrusted suggestion +that must complete its own normal peer handshake before it becomes a +connection. + +### MdnsDiscovery + +`MdnsDiscovery` browses `_numax._tcp.local.` using a cluster-specific DNS-SD +subtype derived from the BLAKE3 hash of the cluster ID. It also requires an +exact `cluster` TXT property match. This two-part filter prevents accidental +cross-cluster discovery; neither value is authentication evidence. + +Resolved instances retain first-observation order. Addresses within an +instance are sorted and deduplicated. The application-owned retained endpoint +contributions are bounded **globally** by `max_candidates`, including duplicate +contributions from different instances, not by `max_instances * max_candidates`. +Replacing an instance reclaims its previous allocation before admission; the +instance count and flattened candidate view are also bounded. Port zero, unspecified and multicast addresses, and +IPv6 link-local addresses without a usable scope are ignored. A DNS-SD removal +event removes the complete instance contribution; expiry is delegated to the +mDNS daemon's cache and removal events. + +These are Numax application-state bounds, not a whole-library memory cap. +`mdns-sd 0.21` does not expose a configurable bound for its internal DNS record +cache; `max_instances` and `max_candidates` do not bound that cache. Do not +interpret them as protection against arbitrary untrusted multicast traffic. + +mDNS announcement support is required. Announcements accept a concrete IP +address or a `.local` hostname, never a wildcard host or port zero. The provider +filters its own DNS-SD names and advertised endpoints. Original registration +keys remain distinct from per-interface aliases reported by DNS-SD name-conflict +events: unregister uses the original key, not the renamed wire alias. +Re-announcement registers a replacement under a distinct original key before +withdrawing the previous registration and awaiting its acknowledgement. +A rejected registration leaves the previous one owned; failed withdrawal stops +the browse loop and starts checked cleanup rather than accumulating more +registrations. At most two original registrations are owned during replacement. +Own-name history (including aliases) and endpoint history each retain at most +1024 entries until daemon termination, so late cached resolutions are still +self-filtered. History exhaustion rejects an announcement or terminates browsing +on a new alias that cannot be retained; it does not silently evict self-filtering +history. Once queued, the browse task owns announcement completion even if the +calling future is cancelled. + +Shutdown has one cleanup owner and one absolute four-second deadline measured +from the first shutdown request. A replacement withdrawal in progress selects +on that request; cancellation retains both original keys for cleanup instead of +losing ownership. Cleanup requests unregister/goodbye for every owned key, stops +browsing, requests daemon shutdown, awaits its acknowledgement, joins the bridge +task, and clears the view. Repeated shutdown calls cannot renew the deadline. +The common budget reserves time for daemon termination even when unregister +fails or its acknowledgement never arrives; queue retries are also bounded by +that same deadline. The coordinator's five-second provider timeout therefore +exceeds the complete provider-owned sequence. Cleanup errors are reported, not +silently treated as success. A daemon acknowledgement does **not** guarantee receipt of a UDP +goodbye by every LAN peer. Drop is best-effort fallback, not a stronger delivery +guarantee. This provider is intended for LAN development and demos, not +untrusted multicast networks. + +### DnsSrvDiscovery + +`DnsSrvDiscovery` reads a fully qualified SRV name beginning with `_` and +ending with `.`, using the system resolver. It starts with an empty view and +performs refreshes in the background. Results are sorted deterministically by +SRV priority, target, port and weight, then deduplicated and bounded. Root +targets and records with port zero do not become candidates. SRV weight is not +used as a membership assertion or a connection authorization rule. + +A successful answer replaces the complete view. Refresh happens no later than +the DNS validity deadline and is capped by `max_refresh_interval`. A successful +empty or no-record answer removes the previous view. A transient lookup error +keeps the last valid view only until its DNS validity deadline, then removes it +while retrying at `retry_interval`. DNS-SRV does not support announcements. +Shutdown cancels an in-flight resolver lookup, then stops and joins the refresh +task. + +### FileWatchDiscovery + +`FileWatchDiscovery` polls an externally managed UTF-8 file. Each trimmed, +non-empty line is one `host:port` endpoint; a line whose first non-whitespace +character is `#` is a comment. Entries are canonicalized and deduplicated in +first-occurrence order. File size, candidate count, event capacity and polling +interval are bounded and configurable. + +A missing file is a valid empty view, both initially and after removal. This +also observes delayed creation and Kubernetes-style atomic file replacement. +The initial read fails for other I/O, encoding, syntax or limit errors. After a +valid snapshot exists, an unreadable, non-UTF-8, malformed, oversized or +over-limit update is rejected atomically and the last valid snapshot remains +active; polling continues. File discovery does not support announcements. +Shutdown stops and joins the polling task. + +### Provider dependencies + +The two added runtime dependencies have narrow protocol roles. `mdns-sd` +provides DNS-SD browse, cache-expiry, unregister/goodbye and daemon shutdown +behavior that should not be reimplemented as ad-hoc multicast parsing. +`hickory-resolver` provides real SRV records and their DNS validity deadlines; +Tokio's host lookup does not expose either. File discovery uses Tokio polling +instead of adding a filesystem-notification dependency, which also makes +delete/create and atomic replacement semantics consistent across platforms. + +## Endpoints, identity, and connection admission + +Four values remain deliberately separate: + +- a discovery candidate is an untrusted endpoint suggestion; +- an advertised endpoint is the address the local node asks providers to + publish; +- a transport address is the actual remote TCP endpoint of an active socket; +- a peer identity is the `NodeId` learned in the handshake together with its + verification level (`CertificateBound` or `Unverified`). + +For outbound connections Numax also retains the candidate that was dialed. An +inbound connection has no dialed candidate. Discovery never promotes an +endpoint into an authenticated identity or an active connection. + +Candidate ports must be non-zero and unspecified IP addresses such as +`0.0.0.0` and `::` are rejected. When the listener uses port zero, an explicit +advertised endpoint with port zero inherits the actual bound port. A wildcard +bind cannot be announced without an explicit non-wildcard advertised host. Both +the concrete bind address and advertised endpoint are excluded from candidates +when available. + +Self endpoints are filtered before dialing, and a connection claiming the +local `NodeId` is rejected after the handshake. Candidate duplicates are +collapsed, concurrent outbound attempts are globally limited to one, and a +second attempt to the same endpoint is rejected while the first is pending. +Active and in-progress connections share the existing `max_peers` semaphore. +Simultaneous connections arriving through different transport addresses remain +distinct and each consumes a slot; no nondeterministic identity-based winner is +selected without a protocol-level connection nonce. + +The default candidate bound is 1024 and is configurable through +`DiscoveryRuntimeConfig`. Reconnect retains its existing per-endpoint backoff +and fatal wire-error policy. Anti-entropy retains its existing bounded op-log +pull and deduplication behavior. + +## Cluster isolation + +Each provider reports the logical cluster it serves. Startup rejects a provider +whose cluster differs from the runtime cluster, and duplicate source IDs are +invalid. Provider implementations must scope all snapshots, changes, and +announcements to that cluster. The cluster value is a discovery routing scope, +not proof of membership and not a replacement for TLS identity or authorization. +The bootstrap handshake carries and validates it; the normal replication +`Hello` remains unchanged. + +## Security and compatibility boundaries + +A discovered endpoint is only a connection candidate. Discovery does not assert +node identity, authenticate a peer, authorize a connection, or establish +membership. Existing TLS and mTLS verification, peer allowlists, connection +limits, and handshake checks remain authoritative when the runtime attempts a +connection. + +Static, mDNS, DNS-SRV and file discovery do not change persisted data or the +WebAssembly host and guest APIs. Bootstrap adds a wire exchange and therefore +increments `PROTOCOL_VERSION` to `5`; version `4` peers are rejected before +bootstrap or replication admission. No storage migration or guest ABI change +is involved. See [Wire Versioning](/numax/design/wire-versioning/) for the exact +compatibility boundary. + +The CLI resolves provider selection from flags, `NX_DISCOVERY_*` variables and +the `[discovery]` TOML section. Explicit peers continue to contribute a static +source when a dynamic provider is selected; they are never reinterpreted as +bootstrap seeds. Provider construction occurs in `nx-core` after the durable +local `NodeId` has been loaded. + +## Verification coverage + +Deterministic unit and component tests cover static compatibility, bounded +watch overflow, snapshot revision continuity, late candidate arrival, +overlapping source contributions, source removal, startup rollback and +cancellation-safe shutdown. Provider-specific tests additionally cover: + +- bootstrap TTL expiry during a stalled seed query, bounded responses, + authenticated TLS/allowlist rejection, seed loss, restart and withdrawal; +- DNS-SRV ordering, filtering, refresh, validity expiry, transient failure, + recovery and cancellation of an in-flight lookup; +- file creation and removal, atomic replacement, malformed and non-UTF-8 + updates, last-good retention, recovery and shutdown; +- mDNS address and instance bounds, self filtering, removal and service-name + conflicts, shutdown during replacement, missing withdrawal acknowledgements, + preservation of both owned keys and non-renewable cleanup deadlines. + +Regression coverage also exercises observation freshness versus cached replay, +resubscription timestamps, global mDNS retained-state bounds, bounded shutdown +acknowledgements, non-blocking startup dialing and anti-entropy over active +connections independently of discovery churn. Capacity tests cover the accepted +maximum, rejection of zero, maximum-plus-one and `usize::MAX`, legacy static +normalization with ordered duplicate peers, and defensive internal channel +rotation. Lifecycle tests cover delayed cleanup as a restart barrier, worker +and cleanup panics, concurrent resubscription, and terminal explicit shutdown. +Test presence is not evidence +that every environment-dependent scenario has run successfully. + +The ignored +`discovery::mdns::tests::two_daemons_discover_and_remove_an_announced_endpoint` +test exercises two real DNS-SD daemons over local multicast, including goodbye +removal. CI runs this check explicitly on a dedicated macOS runner; keeping it ignored prevents the ordinary cross-platform suite from failing on hosts or containers without multicast support. Run it manually on a multicast-capable host with: + +```sh +cargo test -p nx-core \ + discovery::mdns::tests::two_daemons_discover_and_remove_an_announced_endpoint \ + -- --ignored --exact +``` + +CI also explicitly selects +`discovery_lan::mdns_three_daemons_recover_missed_crdt_ops_after_restart` from +the CLI multiprocess suite on macOS, with `NUMAX_MDNS_E2E=1` and +`NUMAX_MDNS_LAN_IP` derived from a real local interface. It builds both reader +and writer variants of the `discovery_lan` guest first. The generic Ubuntu +ignored-test invocation excludes this multicast-specific module. + +That E2E uses three real daemon **processes on one host**, without `--peer`, +and checks discovery, CRDT replication, missed-operation recovery after restart +within a configured 128-operation retention bound, stable identities and +shutdown. It is not evidence of a run on three separate LAN devices or of +recovery beyond retained history. The three-device LAN demo remains a separate +release closing check; neither provider-test presence nor CI wiring asserts it +has passed. diff --git a/docs/nx-site/src/content/docs/design/wire-versioning.md b/docs/nx-site/src/content/docs/design/wire-versioning.md index 1346376..cbb0350 100644 --- a/docs/nx-site/src/content/docs/design/wire-versioning.md +++ b/docs/nx-site/src/content/docs/design/wire-versioning.md @@ -5,11 +5,15 @@ description: Rules for evolving the Numax peer protocol safely. ## Purpose -`PROTOCOL_VERSION` identifies the wire contract used between Numax peers and It is +`PROTOCOL_VERSION` identifies the wire contract used between Numax peers and is independent from the Numax release version. The current value is defined in `crates/nx-net/src/message.rs`. +In `v0.1.5`, the current Numax version, the value is `5`. Version `5` adds the +one-shot bootstrap handshake described below; it is not wire-compatible with +the version `4` protocol shipped by `v0.1.4`. + ## Compatibility policy Numax currently requires an exact version match: @@ -81,3 +85,79 @@ Numax supports bincode and JSON, compatibility must be evaluated for both: is established. Never reuse a protocol version for a different wire contract. + +## Protocol 5 bootstrap exchange + +Protocol `5` adds private wire variants `BootstrapHello` and `BootstrapAck` after +the legacy public `MessageKind` layout and adds a private +`BootstrapRejected` wire error after the legacy public `WireError` layout. A bootstrap +exchange is an alternative one-shot handshake on the normal peer listener; it +does not turn into a replication connection. + +```text +client -> seed: BootstrapHello { + node_id, + protocol_version: 5, + supported_formats, + preferred_format, + cluster_id, + advertised_endpoint?, + max_results +} + +seed -> client: BootstrapAck { + node_id, + protocol_version: 5, + selected_format, + cluster_id, + candidates, + candidate_ttl_ms +} +``` + +The seed validates the exact protocol version, negotiates JSON or Bincode, +authenticates the requester's claimed `NodeId` through the same TLS certificate +binding and allowlist policy as a normal handshake, and requires an exact +cluster ID match. It validates any advertised endpoint before caching it. The +request's `max_results`, the server response limit and the server cache limit +bound the exchange independently. + +The exported `nx_net::MAX_BOOTSTRAP_RESPONSE_CAPACITY` is `4096`. Client and +server response capacities must be in `1..=4096`; the effective CLI +`discovery.max_candidates` obeys this upper bound only in bootstrap mode. +This resource limit is independent of the wire version, package version, cache +capacity and message-byte limit. + +The client validates the seed's protocol version, selected format, cluster ID, +authenticated identity, response length, candidate lease and every endpoint. +Duplicate endpoints, wildcard hosts, port zero and malformed responses reject +the complete response. Successful completion closes the one-shot connection; +it neither registers the seed as an active replication peer nor emits a peer +connection event. + +Only the requester's identity and the responding seed's identity are covered by +that exchange. The returned endpoint strings are untrusted discovery +candidates. Dialing one later requires a new normal `Hello`/`HelloAck`, TLS +identity check, allowlist decision and connection-slot admission. + +### Version 4 boundary + +Normal `Hello` exchanges between versions `4` and `5` carry a readable version +field and are rejected with `ProtocolMismatch` before peer registration or CRDT +traffic. A version `4` decoder does not know the new bootstrap variants and may +close a `BootstrapHello` as an invalid message rather than returning a +structured mismatch; this is still a safe rejection and never admits a peer. +Static peer configuration remains source-compatible but does not make mixed +version `4`/`5` clusters wire-compatible. + +JSON and Bincode round trips cover the complete private version `5` message set. +Bincode golden hashes and direct public/private byte comparisons protect every +legacy public variant, while exact-version and multiprocess compatibility coverage +uses the previous `v0.1.4` binary to verify safe rejection at the normal +handshake boundary. + +CI resolves the previous binary's source from the explicit +`refs/tags/v0.1.4` reference and verifies its peeled commit is +`419d840e2afe780e7ad1f4135e39e9b38a4f30b1` before building it. A branch with the +same short name is not an acceptable substitute. This test checks rejection, +not mixed-version replication or unrestricted recovery after history expiry. diff --git a/docs/nx-site/src/content/docs/getting-started/installation.md b/docs/nx-site/src/content/docs/getting-started/installation.md index 10b9c94..02126bf 100644 --- a/docs/nx-site/src/content/docs/getting-started/installation.md +++ b/docs/nx-site/src/content/docs/getting-started/installation.md @@ -5,7 +5,10 @@ description: Install Numax on Linux, macOS, Windows or with Cargo. Numax installs the `nx` CLI. -For `v0.1.4`, the recommended path is: +`v0.1.5` is the latest Numax version. It includes static, bootstrap, mDNS, +DNS-SRV and file-based peer discovery. + +For `v0.1.5`, the recommended path is: 1. download a prebuilt binary from the GitHub Release; 2. or build/install from source with Cargo. @@ -38,7 +41,7 @@ rustup target add wasm32-unknown-unknown Use the Linux x86_64 musl build: ```bash -VERSION=v0.1.4 +VERSION=v0.1.5 TARGET=x86_64-unknown-linux-musl ARCHIVE="numax-${VERSION}-${TARGET}.tar.gz" @@ -61,7 +64,7 @@ For ARM64 Linux, use `TARGET=aarch64-unknown-linux-musl`. Apple Silicon: ```bash -VERSION=v0.1.4 +VERSION=v0.1.5 TARGET=aarch64-apple-darwin ARCHIVE="numax-${VERSION}-${TARGET}.tar.gz" @@ -78,7 +81,7 @@ nx --version Intel Mac: ```bash -VERSION=v0.1.4 +VERSION=v0.1.5 TARGET=x86_64-apple-darwin ARCHIVE="numax-${VERSION}-${TARGET}.tar.gz" @@ -99,7 +102,7 @@ nx --version Open PowerShell: ```powershell -$Version = "v0.1.4" +$Version = "v0.1.5" $Target = "x86_64-pc-windows-msvc" $Archive = "numax-$Version-$Target.zip" $Base = "https://github.com/GianIac/numax/releases/download/$Version" diff --git a/docs/nx-site/src/content/docs/getting-started/introduction.md b/docs/nx-site/src/content/docs/getting-started/introduction.md index efa392f..7ed1651 100644 --- a/docs/nx-site/src/content/docs/getting-started/introduction.md +++ b/docs/nx-site/src/content/docs/getting-started/introduction.md @@ -65,7 +65,7 @@ collaborative tools, config propagation across nodes, small multiplayer state. More primitives are coming. - **General-purpose database with rich queries** - not what Numax is. - **Critical production workloads** - Numax is at `v0.1.x`, tested and usable, but still early. - The remaining limits are documented in the [Roadmap](/roadmap/). + The remaining limits are documented in the [Roadmap](/numax/roadmap/). These are current limits, not permanent ones. If something is blocking you, [open an issue](https://github.com/GianIac/numax/issues/new) - that's exactly how priorities get shaped. @@ -93,14 +93,14 @@ Sync uses gossip with periodic anti-entropy for recovery. `v0.1.x` - first stable release line, intended for controlled and non-critical workloads. It works. The examples run. The two nodes converge. -The remaining limits are documented in the [Roadmap](/roadmap/). +The remaining limits are documented in the [Roadmap](/numax/roadmap/). --- ## Where to go next -- Never touched Numax before - [Quickstart: 5 Minutes](/getting-started/quickstart-5-min/) -- Want to write a module - [Your First Module](/getting-started/your-first-module/) -- Words like CRDT or gossip are new - [Foundations](/concepts/foundations/) -- Want to understand the full vision - [Whitepaper](/whitepaper/) -- Want to see where the project is going - [Roadmap](/roadmap/) \ No newline at end of file +- Never touched Numax before - [Quickstart: 5 Minutes](/numax/getting-started/quickstart-5-min/) +- Want to write a module - [Your First Module](/numax/getting-started/your-first-module/) +- Words like CRDT or gossip are new - [Foundations](/numax/concepts/foundations/) +- Want to understand the full vision - [Whitepaper](/numax/whitepaper/) +- Want to see where the project is going - [Roadmap](/numax/roadmap/) diff --git a/docs/nx-site/src/content/docs/getting-started/your-first-module.md b/docs/nx-site/src/content/docs/getting-started/your-first-module.md index 9f7f562..756cea6 100644 --- a/docs/nx-site/src/content/docs/getting-started/your-first-module.md +++ b/docs/nx-site/src/content/docs/getting-started/your-first-module.md @@ -16,7 +16,7 @@ Any language that compiles to WASM can be a Numax module. This page shows Rust ## What you need -Numax already built from the [Quickstart](/getting-started/quickstart-5-min/). +Numax already built from the [Quickstart](/numax/getting-started/quickstart-5-min/). If not: ```bash @@ -381,6 +381,6 @@ In the meantime, browse everything already available in the ## Next steps -- Make it distributed - [Quickstart: 5 Minutes](/getting-started/quickstart-5-min/) +- Make it distributed - [Quickstart: 5 Minutes](/numax/getting-started/quickstart-5-min/) - Explore the full SDK: `nx_sdk::crdt`, `nx_sdk::net`, `nx_sdk::system`, `nx_sdk::time` -- Browse the [examples directory](https://github.com/GianIac/numax/tree/main/examples) \ No newline at end of file +- Browse the [examples directory](https://github.com/GianIac/numax/tree/main/examples) diff --git a/docs/nx-site/src/content/docs/guides/debugging-wasm-modules.md b/docs/nx-site/src/content/docs/guides/debugging-wasm-modules.md index e5581e4..2ed1e87 100644 --- a/docs/nx-site/src/content/docs/guides/debugging-wasm-modules.md +++ b/docs/nx-site/src/content/docs/guides/debugging-wasm-modules.md @@ -345,7 +345,7 @@ If the values diverge, `--log-level debug` shows which ops were received and app ## Related -- [WASM execution](/concepts/wasm-execution/) - sandbox, entry point and HostState -- [CRDT and state](/concepts/crdt-and-state/) - how ops are applied and propagated -- [Observability](/guides/observability/) - full metrics and health check setup -- [CLI reference](/reference/cli/) - all available flags \ No newline at end of file +- [WASM execution](/numax/concepts/wasm-execution/) - sandbox, entry point and HostState +- [CRDT and state](/numax/concepts/crdt-and-state/) - how ops are applied and propagated +- [Observability](/numax/guides/observability/) - full metrics and health check setup +- [CLI reference](/numax/reference/cli/) - all available flags diff --git a/docs/nx-site/src/content/docs/reference/cli.md b/docs/nx-site/src/content/docs/reference/cli.md index 7a6170b..24ed35b 100644 --- a/docs/nx-site/src/content/docs/reference/cli.md +++ b/docs/nx-site/src/content/docs/reference/cli.md @@ -43,8 +43,15 @@ Sync is disabled by default. Pass `--listen` to enable it. |---|---|---| | `--listen ` | `NX_LISTEN` | Address to listen on (e.g. `0.0.0.0:9000`). Required for sync | | `--peer ` | `NX_PEER` / `NX_PEERS` | Peer address to connect to. Can be repeated. Requires `--listen` | +| `--discovery-mode ` | `NX_DISCOVERY_MODE` | `static`, `bootstrap`, `mdns`, `dns-srv`, or `file` | +| `--bootstrap-seed ` | `NX_DISCOVERY_SEEDS` | Bootstrap endpoint. Can be repeated | +| `--mdns-instance ` | `NX_DISCOVERY_INSTANCE_NAME` | Local mDNS instance name | +| `--dns-srv-name ` | `NX_DISCOVERY_SERVICE_NAME` | Fully qualified DNS-SRV service name | +| `--peer-file ` | `NX_DISCOVERY_FILE` | Peer file to watch | `NX_PEERS` accepts a comma-separated list: `NX_PEERS=127.0.0.1:9001,127.0.0.1:9002` +Dynamic discovery still requires `--listen`. Explicit `--peer` values remain an +additional static source and are not treated as bootstrap seeds. ### Timing @@ -221,6 +228,13 @@ anti_entropy_interval = "30s" [discovery] mode = "static" +# cluster_id = "default" +# advertised_endpoint = "127.0.0.1:9000" +# max_candidates = 1024 +# Bootstrap: seeds, refresh_interval, retry_initial, retry_max, stale_after, max_seeds +# mDNS: instance_name, max_instances +# DNS-SRV: service_name, retry_interval, max_refresh_interval +# File: path, poll_interval, max_file_bytes ``` ### nx config validate @@ -419,7 +433,11 @@ mode = "static" | Field | Type | Values | Description | |---|---|---|---| -| `mode` | string | `static` | Peer discovery mode. Only `static` is supported today. Dynamic discovery is on the roadmap | +| `mode` | string | `static`, `bootstrap`, `mdns`, `dns-srv`, `file` | Peer discovery provider | + +Provider selectors are `seeds` for bootstrap, `instance_name` for mDNS, +`service_name` for DNS-SRV, and `path` for file discovery. See the +[configuration reference](/numax/reference/config/) for all tuning fields and environment variables. --- diff --git a/docs/nx-site/src/content/docs/reference/config.md b/docs/nx-site/src/content/docs/reference/config.md index dc123f2..a754a68 100644 --- a/docs/nx-site/src/content/docs/reference/config.md +++ b/docs/nx-site/src/content/docs/reference/config.md @@ -86,6 +86,13 @@ anti_entropy_interval = "30s" [discovery] mode = "static" +# cluster_id = "default" +# advertised_endpoint = "127.0.0.1:9000" +# max_candidates = 1024 +# Bootstrap: seeds, refresh_interval, retry_initial, retry_max, stale_after, max_seeds +# mDNS: instance_name, max_instances +# DNS-SRV: service_name, retry_interval, max_refresh_interval +# File: path, poll_interval, max_file_bytes ``` All fields are optional. Unknown fields are rejected at validation time. @@ -256,20 +263,62 @@ anti_entropy_interval = "60s" ## [discovery] -Controls how peers are discovered. +Controls how peers are discovered in `v0.1.5`, the current Numax version. +Dynamic discovery is available alongside backward-compatible static peer lists. | Field | Type | Default | Description | |---|---|---|---| -| `mode` | string | `static` | Discovery mode. Only `static` is supported today | +| `mode` | string | `static` | `static`, `bootstrap`, `mdns`, `dns-srv`, or `file` | +| `cluster_id` | string | `default` | Discovery routing scope; not an authorization boundary | +| `advertised_endpoint` | string | derived from listener | Concrete endpoint published by bootstrap or mDNS | +| `max_candidates` | integer | `1024` | Positive aggregate bound across all discovery sources; at most `4096` in bootstrap mode | -In `static` mode, peers are explicitly listed in `[network].peers` or via `--peer` flags. -Dynamic discovery (mDNS, DNS-SRV, SWIM) is on the roadmap. +Provider-specific fields are accepted only for their selected mode: + +| Mode | Required fields | Optional fields and defaults | +|---|---|---| +| `static` | none | none | +| `bootstrap` | `seeds` | `refresh_interval = "20s"`, `retry_initial = "500ms"`, `retry_max = "30s"`, `stale_after = "2m"`, `max_seeds = 32` | +| `mdns` | `instance_name` | `max_instances = 1024` | +| `dns-srv` | `service_name` | `retry_interval = "5s"`, `max_refresh_interval = "5m"` | +| `file` | `path` | `poll_interval = "2s"`, `max_file_bytes = "1MiB"` | + +Explicit peers from `[network].peers`, `--peer`, `NX_PEER`, or `NX_PEERS` +remain an additional static source when a dynamic mode is selected. They never +become bootstrap seeds. Every non-static mode enables sync and therefore +requires `[network].listen`, `--listen`, or `NX_LISTEN`. + +For compatibility, the effective candidate capacity is raised to at least the +number of explicit peer entries. In bootstrap mode that effective value must +also be at most `4096`: a larger explicit peer list is rejected, not silently +truncated. The bootstrap upper bound does not apply to static, mDNS, DNS-SRV or +file mode. `NX_DISCOVERY_MAX_CANDIDATES` overrides the TOML value; validation uses +the resolved mode and capacity. + +Successful startup means local services are ready, not that discovery has +found peers or CRDT state has converged. Candidate expiry stops new dialing but +does not close admitted connections; periodic anti-entropy continues over those +active connections. Recovery depends on retained operation and deduplication +history, not merely on rediscovery. See the +[discovery contract](/numax/design/discovery-contract/) for freshness, shutdown +and mDNS resource limits. ```toml [discovery] -mode = "static" +mode = "bootstrap" +cluster_id = "production" +advertised_endpoint = "10.0.0.12:9000" +seeds = ["10.0.0.10:9000", "10.0.0.11:9000"] ``` +### Advertised endpoint resolution rules + +The `advertised_endpoint` specifies the dialable address announced to peers through dynamic discovery providers (mDNS, bootstrap gossip, etc.): + +- **Explicit unicast listener**: When `[network].listen` specifies a concrete IP address (e.g., `192.168.1.50:9000`), `advertised_endpoint` defaults to that address and is optional. +- **Wildcard listener (`0.0.0.0` or `[::]`)**: An explicit `advertised_endpoint` is **required** because wildcard addresses are not dialable by remote peers. +- **Dynamic port binding (`:0`)**: If configured with port zero (e.g., `192.168.1.50:0`), Numax automatically resolves the port to the actual ephemeral port assigned by the OS upon binding. + --- ## Environment variables @@ -297,6 +346,23 @@ They are useful for secrets (TLS paths), container environments, and CI. | `NX_MANAGEMENT_REQUEST_TIMEOUT_SECS` | integer | `[management].request_timeout_secs` | HTTP header-read and routed-request timeout in seconds | | `NX_LOG_LEVEL` | string | `[observability].log_level` | `trace`, `debug`, `info`, `warn`, `error` | | `NX_LOG_FORMAT` | string | `[observability].log_format` | `text` or `json` | +| `NX_DISCOVERY_MODE` | string | `[discovery].mode` | `static`, `bootstrap`, `mdns`, `dns-srv`, or `file` | +| `NX_DISCOVERY_CLUSTER_ID` | string | `[discovery].cluster_id` | Discovery routing scope | +| `NX_DISCOVERY_ADVERTISED_ENDPOINT` | string | `[discovery].advertised_endpoint` | Endpoint to publish | +| `NX_DISCOVERY_MAX_CANDIDATES` | integer | `[discovery].max_candidates` | Aggregate candidate bound | +| `NX_DISCOVERY_SEEDS` | CSV | `[discovery].seeds` | Bootstrap seed endpoints | +| `NX_DISCOVERY_REFRESH_INTERVAL` | duration | `[discovery].refresh_interval` | Bootstrap refresh interval | +| `NX_DISCOVERY_RETRY_INITIAL` / `NX_DISCOVERY_RETRY_MAX` | duration | matching fields | Bootstrap retry bounds | +| `NX_DISCOVERY_STALE_AFTER` | duration | `[discovery].stale_after` | Bootstrap candidate lease | +| `NX_DISCOVERY_MAX_SEEDS` | integer | `[discovery].max_seeds` | Bootstrap seed bound | +| `NX_DISCOVERY_INSTANCE_NAME` | string | `[discovery].instance_name` | mDNS instance name | +| `NX_DISCOVERY_MAX_INSTANCES` | integer | `[discovery].max_instances` | mDNS instance bound | +| `NX_DISCOVERY_SERVICE_NAME` | string | `[discovery].service_name` | Fully qualified DNS-SRV name | +| `NX_DISCOVERY_RETRY_INTERVAL` | duration | `[discovery].retry_interval` | DNS retry interval | +| `NX_DISCOVERY_MAX_REFRESH_INTERVAL` | duration | `[discovery].max_refresh_interval` | DNS refresh ceiling | +| `NX_DISCOVERY_FILE` | path | `[discovery].path` | Watched peer file | +| `NX_DISCOVERY_POLL_INTERVAL` | duration | `[discovery].poll_interval` | File polling interval | +| `NX_DISCOVERY_MAX_FILE_BYTES` | byte size | `[discovery].max_file_bytes` | Peer-file size bound | `NX_PEER` and `NX_PEERS` are additive: if both are set, both peers are used. @@ -365,5 +431,5 @@ nx run my_module.wasm --config node-b.toml --settle-for 5s ## Related -- [CLI reference](/reference/cli/) - full flag and subcommand reference -- [Host API](/reference/host-api/) - functions available to WASM modules +- [CLI reference](/numax/reference/cli/) - full flag and subcommand reference +- [Host API](/numax/reference/host-api/) - functions available to WASM modules diff --git a/docs/nx-site/src/content/docs/reference/crates/index.md b/docs/nx-site/src/content/docs/reference/crates/index.md index 1731873..cfd2070 100644 --- a/docs/nx-site/src/content/docs/reference/crates/index.md +++ b/docs/nx-site/src/content/docs/reference/crates/index.md @@ -193,5 +193,5 @@ nx-sdk ──────────────────────── ## Where to go next - [Host API](/numax/reference/host-api/) - the functions `nx-sdk` calls and `nx-core` implements -- [Configuration](/numax/reference/configuration/) - how `nx-cli` resolves config before passing it to `nx-core` +- [Configuration](/numax/reference/config/) - how `nx-cli` resolves config before passing it to `nx-core` - [CLI](/numax/reference/cli/) - the user-facing `nx` command surface diff --git a/docs/nx-site/src/content/docs/reference/crates/nx-cli.md b/docs/nx-site/src/content/docs/reference/crates/nx-cli.md index 8f3dd61..f6297ad 100644 --- a/docs/nx-site/src/content/docs/reference/crates/nx-cli.md +++ b/docs/nx-site/src/content/docs/reference/crates/nx-cli.md @@ -20,7 +20,7 @@ and hands a fully-built `RuntimeConfig` to `nx-core`. It never contains runtime | Read environment variables | `config.rs` - `EnvRunConfig::from_env` | | Resolve precedence (CLI > env > file > defaults) | `config.rs` - `EffectiveRunConfig::resolve` | | Validate flag combinations (TLS, sync, settle) | `config.rs` - `validate_tls_flags`, `validate_settle_mode`, etc. | -| Build runtime and Management API configuration | `config.rs` - `build_sync_config`, `build_tls_config`, `build_observability_config`, `build_management_config` | +| Build runtime, discovery and Management API configuration | `config.rs` - `build_sync_config`, `resolve_discovery_config`, `build_tls_config`, `build_observability_config`, `build_management_config` | | Coordinate daemon and Management API lifecycle | `main.rs` - `Cli::Serve` | | Initialize logging and optional Tokio Console diagnostics | `config.rs` - `init_logging` | | Generate `numax.toml` template | `config.rs` - `CONFIG_TEMPLATE`, `init_config_file` | @@ -93,6 +93,11 @@ pub struct RunCliOptions { pub verbose: bool, pub log_level: Option, pub log_format: Option, + pub discovery_mode: Option, + pub bootstrap_seeds: Vec, + pub mdns_instance: Option, + pub dns_srv_name: Option, + pub peer_file: Option, } ``` @@ -117,6 +122,7 @@ Built by `EnvRunConfig::from_env()`. Each field maps to one env var: | `serialization_format` | `NX_SERIALIZATION_FORMAT` | `bincode` or `json` | | `log_level` | `NX_LOG_LEVEL` | | | `log_format` | `NX_LOG_FORMAT` | `text` or `json` | +| discovery settings | `NX_DISCOVERY_*` | Mode and provider-specific values | **`RunFileConfig`** - what came from `numax.toml`. Sections: @@ -143,6 +149,7 @@ pub struct EffectiveRunConfig { pub sync: Option, pub observability: Option, pub management: Option, + pub discovery: RuntimeDiscoveryConfig, pub log_level: String, pub log_format: LogFormat, } @@ -158,9 +165,9 @@ Sync is not always enabled. `build_sync_config` decides: - If any sync-related field is present (env, file, TLS, format) but `listen` is missing β†’ **error**. Dialer-only mode is not supported. - If `listen` is set β†’ sync enabled, `SyncConfig` is built and returned. -`force_enabled` is `true` when the config file has `[network]`, `[tls]`, or `[limits]` sections, -or when env vars provide sync inputs. This makes `nx config show --effective` work correctly -even without CLI `--listen`. +`force_enabled` is also `true` for dynamic discovery. Dynamic modes therefore +require a listen address. Explicit peers remain a separate static provider and +are composed with the selected dynamic provider. --- @@ -296,6 +303,6 @@ cargo test -p nx-cli Use this page together with the user-facing CLI and config docs: - [CLI reference](/numax/reference/cli/) - flags and subcommands exposed by `nx` -- [Configuration](/numax/reference/configuration/) - TOML and environment variable reference +- [Configuration](/numax/reference/config/) - TOML and environment variable reference - [nx-core crate](/numax/reference/crates/nx-core/) - the runtime layer `nx-cli` calls into - [Crates overview](/numax/reference/crates/) - where `nx-cli` fits in the dependency graph diff --git a/docs/nx-site/src/content/docs/reference/crates/nx-core.md b/docs/nx-site/src/content/docs/reference/crates/nx-core.md index 2de54d6..1bed576 100644 --- a/docs/nx-site/src/content/docs/reference/crates/nx-core.md +++ b/docs/nx-site/src/content/docs/reference/crates/nx-core.md @@ -22,6 +22,7 @@ Everything below that boundary lives here or in the crates it composes. | Remote operation application | `sync_manager/apply.rs` | | Durable CRDT state and startup hydration | `sync_manager/storage.rs` | | Anti-entropy, peer broadcast and reconnect handling | `sync_manager/replication.rs` + `nx-net` | +| Peer discovery contract and candidate coordination | `discovery.rs`, `sync_manager/candidates.rs` | | Schema headers and offline migration support | `sync_manager/schema.rs`, `sync_manager/migration.rs` | | Peer health tracking | `sync_manager/peer.rs` | | NodeId persistence | `runtime.rs` - `load_or_create_node_id` | @@ -98,7 +99,7 @@ Runtime::new(config) | `new(config)` | Opens sled store, builds wasmtime engine + linker with all host API functions registered, creates `SyncManager` if configured | | `start_observability()` | Binds the HTTP metrics endpoint. No-op if not configured | | `start_sync()` | Calls `SyncManager::start()`, starts TCP listener + dial loop. No-op if sync disabled | -| `wait_before_run(dur)` | Repeatedly reconnects configured peers until the deadline. No-op if sync disabled | +| `wait_before_run(dur)` | Repeatedly reconnects current discovery candidates until the deadline. No-op if sync disabled | | `run_module(bytes)` | Compiles or retrieves cached module, builds `HostState`, instantiates, calls `run()` or `_start()` | | `control_handle()` | Returns the shared introspection and management handle used by transport adapters | | `settle_for(dur)` | Sleeps for `dur`, keeping sync alive. No-op if sync disabled | @@ -187,6 +188,106 @@ Peers alone do not enable sync - a node must also listen. `SyncManager` owns the runtime side of replication. It is the bridge between host API calls from guest modules and the network layer in `nx-net`. +The default constructor wraps configured peers in `StaticDiscovery` and remains +backward-compatible. Integrations can use `SyncManager::try_new_with_discovery` +with named `DiscoveryProvider` values and `DiscoveryRuntimeConfig`. The manager +keeps one bounded candidate snapshot shared by initial connection, reconnect and +anti-entropy, while `SyncHandle::active_connections()` exposes transport and +identity-verification details separately. + +### Peer discovery API + +`nx-core` publicly exports the discovery contract and all five initial +providers: + +| Provider | Constructor input | Announcement | Update/removal source | +|---|---|---|---| +| `StaticDiscovery` | `Vec` | unsupported | immutable | +| `BootstrapGossipDiscovery` | seed config + `BootstrapClientConfig` | required | seed refresh and bounded lease expiry | +| `MdnsDiscovery` | instance and cluster config | required | DNS-SD resolve/remove events | +| `DnsSrvDiscovery` | fully qualified SRV name | unsupported | DNS TTL refresh, empty response or expiry | +| `FileWatchDiscovery` | peer-file path | unsupported | periodic complete-file replacement | + +Each `DiscoveryProvider` has a unique source ID and may add a coordinator-level +candidate TTL. `DiscoveryRuntimeConfig` supplies the cluster ID, optional local +advertised endpoint and aggregate candidate bound. Its defaults are cluster +`default`, no explicit advertised endpoint and 1024 candidates. Providers with +required announcement support make sync startup fail when the bound listener +cannot yield a concrete advertised endpoint. + +`DiscoveryWatch` bundles an atomic snapshot with its subsequent bounded event +stream. Dynamic providers use one `DiscoveryChange::Observed` revision for a +complete ordered replacement with per-endpoint observation timestamps, rather +than publishing a temporary empty list. Lag or a revision gap invalidates the watch explicitly; +the coordinator resubscribes and atomically installs the new bundled snapshot. + +Provider-specific defaults are: + +| Provider | Refresh/retry defaults | Provider bounds | +|---|---|---| +| Bootstrap | refresh 20s; retry 500ms to 30s; stale after 120s | 32 seeds; 1024 candidates; 128 events | +| mDNS | daemon-driven TTL/removal | 1024 instances; 1024 candidates; 128 events | +| DNS-SRV | retry 5s; maximum refresh interval 300s | 1024 candidates; 128 events | +| File | poll 2s | 1 MiB file; 1024 candidates; 128 events | + +#### Event capacity API + +`DEFAULT_DISCOVERY_EVENT_CAPACITY` (128) and `MAX_DISCOVERY_EVENT_CAPACITY` +(4096) are public in both `nx_core::discovery` and the crate root. The +`event_capacity` fields in `BootstrapGossipDiscoveryConfig`, +`MdnsDiscoveryConfig`, `DnsSrvDiscoveryConfig` and `FileWatchDiscoveryConfig` +accept only `1..=MAX_DISCOVERY_EVENT_CAPACITY`. Their provider constructors +return `DiscoveryError::InvalidConfiguration` for zero or larger values, +including `usize::MAX`, before allocating channels/state, starting work or +performing provider I/O. mDNS applies the same capacity to announcement requests. +The limit counts event slots, not candidates or total bytes; Tokio may round +broadcast capacity up to a power of two, still no larger than 4096. + +| Static constructor | Result and capacity policy | +|---|---| +| `StaticDiscovery::new(peers)` | `Self`, default capacity 128 | +| `StaticDiscovery::with_event_capacity(peers, capacity)` | `Self`, clamps to `[1, 4096]`; zero becomes one, oversized values become 4096 | +| `StaticDiscovery::try_with_event_capacity(peers, capacity)` | `Result`, rejects capacity outside `1..=4096` with `InvalidConfiguration` before channel allocation | + +All static constructors preserve peer order and duplicates without truncation. +Capacity is a Rust provider API setting, not an additional CLI/TOML field. + +Bootstrap uses the same `NodeId`, TLS configuration, message-size limit, socket +timeout and serialization policy as the runtime when its +`BootstrapClientConfig` is built. It authenticates the seed, but its returned +endpoints remain candidates that pass the normal connection handshake later. +mDNS scopes browse and announcement by cluster, DNS-SRV relies on the supplied +record name, and file/static providers report their configured runtime cluster. +In every case discovery scope is separate from TLS identity and allowlist +authorization. + +The coordinator owns provider lifecycle. It starts watches before binding the +listener, announces only after the actual bound address is known, rolls back +providers and the listener on partial startup, and invokes every provider's +idempotent shutdown hook. Bootstrap withdrawal and mDNS goodbye are attempted +during shutdown; provider tasks are joined within the runtime's bounded +operation policy. + +Explicit `request_shutdown()`/`shutdown()` is terminal for dynamic providers. +Unexpected exit may instead be recovered by a later discovery/watch operation, +but only after the old worker is joined and its cleanup completes successfully; +a finished worker or invalidated watch alone does not authorize restart. Fatal +worker errors and cleanup failures block restart. Cleanup remains owned if a +shutdown waiter is cancelled. Bootstrap conservatively tracks seeds before an +advertising query is awaited, including queries whose responses never arrive; +withdrawal is bounded best effort and does not promise remote delivery. mDNS +reports daemon cleanup acknowledgement errors, but an acknowledgement likewise +does not prove every LAN peer received the goodbye. + +`Runtime::new_with_discovery` accepts the resolved `RuntimeDiscoveryConfig` +after the durable `NodeId` is loaded, then constructs the selected provider. +The bootstrap client inherits the runtime TLS, message-size, socket-timeout and +serialization settings. `Runtime::new` remains the backward-compatible static +constructor for Rust embedders. + +For exact snapshot, expiry, ordering and security semantics, see the +[Peer Discovery Contract](/numax/design/discovery-contract/). + Since `v0.1.1`, its implementation is split by responsibility under `sync_manager/`: orchestration in `manager.rs`, remote application in `apply.rs`, replication in `replication.rs`, persistence in `storage.rs`, peer health in `peer.rs`, and persisted diff --git a/docs/nx-site/src/content/docs/reference/crates/nx-net.md b/docs/nx-site/src/content/docs/reference/crates/nx-net.md index 6db326e..ff028c5 100644 --- a/docs/nx-site/src/content/docs/reference/crates/nx-net.md +++ b/docs/nx-site/src/content/docs/reference/crates/nx-net.md @@ -25,6 +25,8 @@ It depends on `nx-sync` for `Op` and `NodeId` types. It does not depend on `nx-c | Peer slot enforcement (semaphore) | `node.rs` - `connection_slots`, `ensure_peer_slot_available` | | Broadcast and targeted op send | `node.rs` - `Node::broadcast_ops`, `Node::send_ops_to_addr` | | Anti-entropy pull requests | `node.rs` - `Node::send_pull_since_to_addr` | +| Authenticated one-shot bootstrap exchange | `bootstrap.rs`, `node.rs` - `BootstrapClient`, inbound bootstrap handling | +| Bounded bootstrap advertisement cache | `bootstrap.rs` - `BootstrapServerConfig`, `BootstrapServer` | | Cooperative shutdown via watch channel | `node.rs` - `Node::shutdown`, `shutdown_tx` | | Wire message types and encode/decode | `message.rs` - `Message`, `MessageKind` | | Peer state tracking | `node.rs` - `PeerConnection`, `peer.rs` - `PeerInfo`, `PeerState` | @@ -61,13 +63,29 @@ NodeConfig::new(node_id, "0.0.0.0:9000") .with_event_channel_capacity(1024) ``` +`NodeConfig::validate()` checks limits before channel/semaphore allocation or +network startup: `max_peers` cannot exceed Tokio's semaphore capacity, +`event_channel_capacity` must be positive and within that capacity, and +`socket_timeout` must be positive and form a representable deadline. +`max_peers = 0` is valid and disables connection admission. + +Prefer `Node::try_new(config)`, which returns `NodeConfigError` for invalid limits +without binding sockets. The legacy infallible `Node::new(config)` retains its +0.1.4 signature and behavior for source compatibility. Validation does not +establish that a listen address can be bound or a peer reached. + ### Node lifecycle ``` -Node::new(config) +Node::try_new(config)? validate before constructing; no socket binding yet +Node::try_new_with_bootstrap_server(config, bootstrap)? + validate and enable the one-shot bootstrap service └── take_event_receiver() take the event channel before starting └── start_listener() bind TCP, spawn listener task, returns bound SocketAddr └── connect_to_peer(addr) dial, TLS, handshake, register, spawn read loop + └── connection_info(addr) transport, direction and verified/claimed identity + └── announce_bootstrap_endpoint(addr) publish the local bootstrap suggestion + └── withdraw_bootstrap_endpoint() remove that suggestion ...running... └── broadcast_ops(ops) push ops to all connected peers └── send_ops_to_addr(addr, ops) @@ -99,18 +117,18 @@ of the `Node` so the sync manager owns it. ``` connect_to_peer(addr) - 1. acquire semaphore slot (PeerLimitReached if full) - 2. TCP connect with socket_timeout - 3. TLS handshake (if configured) - 4. capture peer_cert DER bytes - 5. send Hello { node_id, protocol_version, supported_formats, preferred_format } - 6. receive HelloAck { node_id, protocol_version, selected_format } - 7. validate protocol version == PROTOCOL_VERSION (4) - 8. if TLS and not insecure: derive NodeId from peer cert, verify == claimed node_id - 9. if allowlist configured: verify peer_node_id in allowed_peers - 10. insert PeerConnection into peers map - 11. emit PeerConnected event - 12. spawn read_loop task + 1. reject a duplicate attempt for the same endpoint and acquire the bounded outbound-attempt slot + 2. acquire connection semaphore slot (PeerLimitReached if full) + 3. TCP connect with socket_timeout and retain the actual transport address + 4. TLS handshake (if configured) + 5. capture peer_cert DER bytes + 6. send Hello { node_id, protocol_version, supported_formats, preferred_format } + 7. receive HelloAck { node_id, protocol_version, selected_format } + 8. validate protocol version == PROTOCOL_VERSION (5) and reject the local NodeId + 9. if TLS and not insecure: derive NodeId from peer cert, verify == claimed node_id + 10. if allowlist configured: verify peer_node_id in allowed_peers + 11. insert PeerConnection and its `PeerConnectionInfo` into the peers map + 12. emit PeerConnected event and spawn the read loop ``` ### Inbound (listener) @@ -118,16 +136,18 @@ connect_to_peer(addr) ``` handle_incoming(stream, addr, context) 1. TLS accept (if configured), capture peer_cert - 2. receive Hello - 3. validate protocol version - 4. negotiate_serialization_format - 5. TLS identity binding (same as outbound) - 6. send HelloAck { node_id, protocol_version, selected_format } - 7. insert PeerConnection into peers map - 8. emit PeerConnected event - 9. run read_loop inline (not spawned - task already spawned by listener) + 2. receive Hello or BootstrapHello + 3. validate protocol version and negotiate_serialization_format + 4. reject the local NodeId, then perform TLS identity binding and allowlist checks + 5a. normal: send HelloAck, insert PeerConnection, emit PeerConnected, run read_loop + 5b. bootstrap: validate service/cluster/request, send BootstrapAck, close without peer admission ``` +`PeerConnectionInfo` keeps the TCP transport address separate from the outbound +endpoint that was dialed. It also records inbound/outbound direction and whether +the handshake NodeId was certificate-bound or unverified. These are runtime +facts only and do not change the wire format. + --- ## Wire format @@ -140,12 +160,15 @@ Every message is framed as: - Length is the total of `format byte + payload`, encoded as big-endian `u32`. - Format byte: `0x01` = JSON, `0x02` = bincode. -- Payload is the serialized `Message` struct. +- Payload is the serialized internal protocol message. For normal replication + frames its layout is byte-for-byte equivalent to the public `Message`; the + bootstrap-only variants remain private so the exhaustive public enums retain + their 0.1.4 source-compatible shape. -`PROTOCOL_VERSION = 4`. Version mismatch during handshake causes a structured +`PROTOCOL_VERSION = 5`. Version mismatch during a recognized handshake causes a structured `WireError::ProtocolMismatch` and immediate disconnect. -### MessageKind variants +### Protocol message variants | Variant | Direction | Purpose | |---|---|---| @@ -156,6 +179,8 @@ Every message is framed as: | `PullSince` | both | Request ops since a known op id (anti-entropy) | | `Ping` / `Pong` | both | Keepalive | | `Error` | both | Structured wire error: `ProtocolMismatch`, `OpRejected`, `RateLimited`, `NotAuthorized`, `Internal` | +| `BootstrapHello` *(private wire variant)* | client -> seed | One-shot identity, format, cluster, optional endpoint advertisement and result limit | +| `BootstrapAck` *(private wire variant)* | seed -> client | Seed identity, format, cluster, bounded candidates and lease | ### WireError semantics @@ -165,6 +190,7 @@ Every message is framed as: | `NotAuthorized` | Fatal for that peer/config | Credentials, certificate identity, or allowlist must change before retrying. | | `RateLimited` | Retryable | Back off. Use `retry_after_ms` when present, otherwise use normal reconnect backoff. | | `OpRejected` | Fatal for those ops | Do not resend the same rejected ops unchanged. Current generic error handling closes the peer connection. | +| `BootstrapRejected` *(private wire error)* | Fatal for that request | Bootstrap is disabled or its cluster, advertisement or request bounds are invalid. | | `Internal` | Retryable with backoff | Treat as transient unless it repeats; record metrics/logs. | The configured-peer reconnect loop uses this policy: fatal wire errors stop @@ -186,6 +212,31 @@ HelloAck selected_format = Json A `--debug-protocol` node (JSON only) always negotiates JSON with any peer. A standard node advertises both and prefers bincode. +### Bootstrap transport + +`BootstrapClient::query(seed, request)` opens a bounded, one-shot connection, +sends `BootstrapHello`, authenticates the `BootstrapAck` seed identity and +returns `BootstrapResponse`. `BootstrapClientConfig` reuses `NodeId`, optional +`TlsConfig`, message-size, socket-timeout and serialization controls. Its +defaults permit one concurrent query, at most 128 returned candidates and a +maximum accepted candidate TTL of 300s. + +The server is enabled through `Node::try_new_with_bootstrap_server`. Keeping the +bootstrap policy outside `NodeConfig` preserves source compatibility with 0.1.4 +struct literals. By default it +retains at most 1024 authenticated requester advertisements for 60s and returns +at most 128 candidates. Its own advertised endpoint is returned first, followed +by cached requester endpoints in stable insertion order; responses are +deduplicated and exclude the current requester. A request without an advertised +endpoint withdraws that requester's cache entry. + +Cluster IDs, endpoint strings, response count and leases are validated on both +sides. A bootstrap socket holds an inbound connection slot while the request is +processed but is never inserted into the active peer map, never enters a read +loop and never emits `PeerConnected`. Suggestions in `BootstrapResponse` are +not authenticated identities; the normal dialer must authenticate each one in +a separate `Hello`/`HelloAck` exchange. + --- ## TLS and mTLS @@ -265,6 +316,7 @@ Node::shutdown() 3. for each task: timeout(3s, task).await - if task does not finish in 3s: task.abort() 4. peers.clear() -> drops all PeerConnection -> drops all semaphore permits + 5. clear the bootstrap advertisement and leased requester cache ``` Read loops check the shutdown signal on every iteration via `tokio::select!`. @@ -279,10 +331,12 @@ This avoids waiting for socket timeouts during clean shutdown. pub enum NetError { Io(std::io::Error), Serialization(serde_json::Error), - BincodeSerialization(Box), + BinarySerialization(wincode::WriteError), + BinaryDeserialization(wincode::ReadError), ConnectionFailed(String), PeerDisconnected(String), InvalidMessage(String), + Wire(WireError), MessageTooLarge { len: usize, limit: usize }, Timeout, ChannelClosed, @@ -293,6 +347,13 @@ pub enum NetError { } ``` +Configuration validation uses the additive, `#[non_exhaustive]` +`NodeConfigError`. Bootstrap-only construction and queries use the additive, +`#[non_exhaustive]` `BootstrapError`, which distinguishes invalid configuration, +query concurrency, authenticated rejection, invalid responses, node +configuration and transport failures. `NetError` retains exactly its 0.1.4 +variants so existing exhaustive matches remain source-compatible. + --- ## Defaults @@ -303,6 +364,11 @@ pub enum NetError { | `DEFAULT_MAX_MESSAGE_SIZE` | 16 MiB | Maximum wire message size | | `DEFAULT_SOCKET_TIMEOUT` | 30s | Read/write timeout per operation | | `DEFAULT_EVENT_CHANNEL_CAPACITY` | 1024 | Event channel buffer size | +| `DEFAULT_BOOTSTRAP_CACHE_CAPACITY` | 1024 | Seed-side advertised endpoint cache | +| `DEFAULT_BOOTSTRAP_RESPONSE_CAPACITY` | 128 | Results returned by one bootstrap exchange | +| `MAX_BOOTSTRAP_RESPONSE_CAPACITY` | 4096 | Hard limit for one bootstrap response, not the combined discovery snapshot | +| `DEFAULT_BOOTSTRAP_CANDIDATE_TTL` | 60s | Seed-side advertisement lease | +| `DEFAULT_MAX_CONCURRENT_BOOTSTRAP_QUERIES` | 1 | Simultaneous queries per bootstrap client | | `TASK_SHUTDOWN_GRACE` | 3s | Cooperative shutdown grace per task | --- @@ -329,6 +395,9 @@ Tests live in `node.rs` and `message.rs` (`#[cfg(test)]`), plus integration test | `connect_to_peer_times_out_during_tls_handshake` | Timeout during TLS handshake | | `connect_to_peer_rejects_protocol_version_mismatch` | old version in HelloAck | | `incoming_rejects_protocol_version_mismatch` | old version in Hello | +| `protocol_v5_binary_encoding_matches_bincode_golden_hashes` | stable binary encoding for normal and bootstrap messages | +| `one_shot_query_returns_candidates_without_registering_a_peer` | bootstrap response without active peer admission or events | +| `cluster_mismatch_is_rejected_without_populating_the_cache` | cluster isolation before advertisement caching | | `incoming_idle_handshake_consumes_peer_slot` | slot held before handshake completes | | `incoming_idle_tls_handshake_releases_peer_slot_after_timeout` | slot released after timeout | | `active_peer_shutdown_does_not_wait_for_socket_timeout` | cooperative shutdown timing | @@ -349,5 +418,5 @@ Use this page together with the sync model and runtime docs: - [nx-sync crate](/numax/reference/crates/nx-sync/) - `Op` and `NodeId` types used by the wire protocol - [nx-core crate](/numax/reference/crates/nx-core/) - the sync manager that drives `Node` -- [Configuration](/numax/reference/configuration/) - TLS fields and limits that become `NodeConfig` +- [Configuration](/numax/reference/config/) - TLS fields and limits that become `NodeConfig` - [Crates overview](/numax/reference/crates/) - where `nx-net` fits in the dependency graph diff --git a/docs/nx-site/src/content/docs/reference/crates/nx-store.md b/docs/nx-site/src/content/docs/reference/crates/nx-store.md index 9a46f3f..b5aad39 100644 --- a/docs/nx-site/src/content/docs/reference/crates/nx-store.md +++ b/docs/nx-site/src/content/docs/reference/crates/nx-store.md @@ -221,4 +221,4 @@ Use this page together with the runtime and user-facing storage docs: - [Crates overview](/numax/reference/crates/) - where `nx-store` fits in the dependency graph - [nx-core crate](/numax/reference/crates/nx-core/) - opens and shares the `Store` - [Host API](/numax/reference/host-api/) - `db_*` functions that call into the store through `nx-core` -- [Configuration](/numax/reference/configuration/) - `[storage].datastore_path` that becomes the store path +- [Configuration](/numax/reference/config/) - `[storage].datastore_path` that becomes the store path diff --git a/docs/nx-site/src/content/docs/roadmap/index.md b/docs/nx-site/src/content/docs/roadmap/index.md index dde11bf..954f94a 100644 --- a/docs/nx-site/src/content/docs/roadmap/index.md +++ b/docs/nx-site/src/content/docs/roadmap/index.md @@ -24,7 +24,7 @@ description: Current status and planned versions. ## Status and goal -- **Current release line**: `v0.1.4` (active - Management API) +- **Latest version**: `v0.1.5` (Peer Discovery - Foundations). - **Final goal of the cycle**: stable `v0.2.0`. - **Philosophy of intermediate releases**: every `0.1.x` is a **stable and usable** release. Capabilities are added incrementally without sacrificing quality. @@ -48,7 +48,7 @@ Unlike `v0.1.0` (declared for non-critical workloads), `v0.2.0` must guarantee: | `v0.1.2` | Performance & Profiling | released | | `v0.1.3` | Supply Chain & Fuzzing | released | | `v0.1.4` | Management API | released | -| `v0.1.5` | Peer Discovery - Foundations | active | +| `v0.1.5` | Peer Discovery - Foundations | current | | `v0.1.6` | Peer Discovery - SWIM & Gossip K-fanout | planned | | `v0.1.7` | Reactive Module Model - Events | planned | | `v0.1.8` | Reactive Module Model - HTTP & Hot Reload | planned | @@ -62,7 +62,7 @@ Unlike `v0.1.0` (declared for non-critical workloads), `v0.2.0` must guarantee: | `v0.2.0-rc.1` | Release Candidate hardening | planned | | `v0.2.0` | **Stable - production-ready, any criticality** | final goal | -> **Legend**: released = previous stable release; active = current release line; planned = future work; final goal = end of the cycle. +> **Legend**: released = previous stable release; current = latest stable release; planned = future work; final goal = end of the cycle. --- @@ -164,47 +164,57 @@ single further CLI command. ## v0.1.5 - Peer Discovery: Foundations 🌐 +**Release status**: current version. The NAT/WAN decision remains open and may +be evaluated ASAP; this release does not introduce a traversal design or +implementation. Verification coverage and its limits are recorded below. + **Goal**: stop requiring `--peer 1.2.3.4:9000` for every node. Introduce discovery providers and bootstrap address exchange; SWIM membership and K-fanout data gossip follow in `0.1.6`. **Abstraction**: -- [ ] `PeerDiscovery` trait with `discover()`, `announce()`, `watch()` methods -- [ ] Internal replacement of `--peer` with a `StaticDiscovery` implementing the trait -- [ ] Define snapshot/watch consistency, provider errors, announcement support, cancellation and bounded event delivery +- [x] `PeerDiscovery` trait with `discover()`, `announce()`, `watch()` methods +- [x] Internal replacement of `--peer` with a `StaticDiscovery` implementing the trait +- [x] Define snapshot/watch consistency, provider errors, announcement support, cancellation and bounded event delivery ([contract](/numax/design/discovery-contract/)) **Peer coordination and identity**: -- [ ] Updateable peer candidates shared with reconnection and anti-entropy, including startup with an empty peer list -- [ ] Distinguish discovery candidates, authenticated identities, advertised listening endpoints and active connections -- [ ] Define duplicate and self-peer handling, simultaneous connections, source expiry and removal semantics -- [ ] Bound candidates, concurrent connection attempts and connections; preserve backoff, TLS identity checks and authorization -- [ ] Define cluster isolation and advertised endpoint validation, including wildcard binds and dynamically assigned ports -- [ ] Own and stop all discovery tasks; roll back partial startup and withdraw announcements on shutdown +- [x] Updateable peer candidates drive initial dialing and reconnection, including startup with an empty peer list; anti-entropy runs over active connections independently of discovery churn +- [x] Distinguish discovery candidates, authenticated identities, advertised listening endpoints and active connections +- [x] Define duplicate and self-peer handling, simultaneous connections, source expiry and removal semantics +- [x] Bound candidates, concurrent connection attempts and connections; preserve backoff, TLS identity checks and authorization +- [x] Define cluster isolation and advertised endpoint validation, including wildcard binds and dynamically assigned ports +- [x] Own and stop all discovery tasks; roll back partial startup and withdraw announcements on shutdown ([contract](/numax/design/discovery-contract/)) **Initial implementations**: -- [ ] `StaticDiscovery` - peer list from config (backward-compatible) -- [ ] `BootstrapGossipDiscovery` - contact a seed and learn bounded lists of advertised endpoints through the handshake/bootstrap exchange; suggestions remain candidates to authenticate, not membership assertions -- [ ] `MdnsDiscovery` - LAN discovery for demo and dev -- [ ] `DnsSrvDiscovery` - discovery via DNS-SRV record -- [ ] `FileWatchDiscovery` - peer file updated externally (useful for K8s headless services) +- [x] `StaticDiscovery` - peer list from config (backward-compatible) +- [x] `BootstrapGossipDiscovery` - contact a seed and learn bounded lists of advertised endpoints through the handshake/bootstrap exchange; suggestions remain candidates to authenticate, not membership assertions +- [x] `MdnsDiscovery` - LAN discovery for demo and dev +- [x] `DnsSrvDiscovery` - discovery via DNS-SRV record +- [x] `FileWatchDiscovery` - peer file updated externally (useful for K8s headless services) **Configuration**: -- [ ] `[discovery]` section in `numax.toml` with `mode = "static" | "bootstrap" | "mdns" | "dns-srv" | "file"` -- [ ] Define provider-specific settings and interaction with explicit peers; preserve CLI > `NX_*` > TOML > defaults and effective-config output +- [x] `[discovery]` section in `numax.toml` with `mode = "static" | "bootstrap" | "mdns" | "dns-srv" | "file"` +- [x] Define provider-specific settings and interaction with explicit peers; preserve CLI > `NX_*` > TOML > defaults and effective-config output **Protocol compatibility**: -- [ ] Specify bootstrap messages and endpoint advertisement; increment the wire version for incompatible changes -- [ ] Verify JSON and Bincode encoding, handshake limits and safe rejection against `v0.1.4`; static configuration compatibility does not imply mixed-version wire compatibility +- [x] Specify bootstrap messages and endpoint advertisement; increment the wire version for incompatible changes +- [x] Verify JSON and Bincode encoding, handshake limits and safe rejection against `v0.1.4`; static configuration compatibility does not imply mixed-version wire compatibility **Explicit decision**: - [ ] Document `nat-traversal.md` - NAT/WAN traversal to be evaluated for `0.2.0`. **Acceptance tests**: -- [ ] Deterministic provider tests for late arrivals, overlapping sources, removals, transient errors, event overflow and shutdown -- [ ] Static configuration regression coverage; bootstrap recovery after seed loss; DNS refresh/expiry; file replacement and malformed updates -- [ ] Real LAN mDNS checks, TLS rejection and reconnection after restart; justify and validate additional provider dependencies +- [x] Deterministic provider tests for late arrivals, overlapping sources, removals, transient errors, event overflow and shutdown +- [x] Static configuration regression coverage; bootstrap recovery after seed loss; DNS refresh/expiry; file replacement and malformed updates +- [x] Automate the environment-gated LAN mDNS check alongside the existing TLS rejection and reconnection-after-restart coverage; provider dependencies are justified in the discovery contract **Closing criterion**: > All five providers pass their acceptance tests. Three nodes on the same LAN discover each other via mDNS without any `--peer` flag, replicate a CRDT update and recover after reconnection within the declared retention window. Reproducible demo in `examples/discovery_lan/`. +**Verification status (2026-09-14)**: the demo and environment-gated three-process +E2E are present. The local macOS run passed discovery, CRDT replication and +restart recovery within a 128-operation retention bound. This same-host test +does not attest a three-device LAN run or the remote cross-platform CI matrix. +The NAT/WAN decision above remains open. + --- ## v0.1.6 - Peer Discovery: SWIM & Gossip K-fanout πŸ•Έ diff --git a/docs/nx-site/src/content/docs/whitepaper/index.md b/docs/nx-site/src/content/docs/whitepaper/index.md index 49ea3f2..de64055 100644 --- a/docs/nx-site/src/content/docs/whitepaper/index.md +++ b/docs/nx-site/src/content/docs/whitepaper/index.md @@ -5,7 +5,7 @@ description: Numax vision, architecture and principles. > **Note** -> This whitepaper is aligned with **v0.1.4**, the current stable Numax release. +> This whitepaper describes **v0.1.5**, the latest stable Numax version. > Compared to previous drafts, most of the `TODO`s have been resolved based on the code present in the repository. What remains open is explicitly labeled as *(Planned)* and tracked in the roadmap. > > **Status labels (consistent with the code):** @@ -13,7 +13,7 @@ description: Numax vision, architecture and principles. > - **(Prototype)**: partially present; internal wiring or critical paths already verified, but not yet production-ready. > - **(Planned)**: foreseen in the roadmap, not yet implemented. > -> **Version reference**: `v0.1.4` - the Management API release for controlled, non-critical workloads. It retains the versioning, profiling and supply-chain foundations of earlier releases and adds authenticated node management, a persistent local module registry and bounded one-shot execution through HTTP. +> **Version reference**: `v0.1.5` - the Peer Discovery: Foundations release for controlled, non-critical workloads. It adds static, bootstrap, mDNS, DNS-SRV and file discovery while retaining authenticated node management, persistent module registration, bounded one-shot execution through HTTP, and the versioning, profiling and supply-chain foundations of earlier releases. > > **Reference roadmap:** future work is tracked by release line and milestone in [Roadmap](/numax/roadmap/). @@ -171,7 +171,7 @@ The separation keeps responsibilities clear and allows components to evolve inde ### 4.2 Supported environments -Numax `v0.1.4` is designed to run as a native runtime on: +Numax `v0.1.5` is designed to run as a native runtime on: - servers (x86_64, ARM64), - edge nodes, @@ -464,8 +464,11 @@ data. Bincode is the default production format; JSON is selected with | `PullSince` | Client β†’ Server | Requests operations after a given OpId | | `Ping` | Bidirectional | Keepalive | | `Pong` | Bidirectional | Response to Ping | +| `Error` | Bidirectional | Structured rejection or failure | +| `BootstrapHello` | Client β†’ Seed | One-shot authenticated bootstrap request, cluster and optional endpoint advertisement | +| `BootstrapAck` | Seed β†’ Client | Seed identity, cluster, negotiated format and bounded endpoint suggestions with a lease | -**Protocol versioning:** version number (`PROTOCOL_VERSION = 4`) exchanged during the handshake. Version mismatches are rejected during handshake to avoid mixed-version wire ambiguity. +**Protocol versioning:** version number (`PROTOCOL_VERSION = 5`) exchanged during the handshake. Version mismatches are rejected during handshake to avoid mixed-version wire ambiguity. Version `4` belongs to Numax `v0.1.4` and is not wire-compatible with `v0.1.5`. **Current status:** @@ -475,7 +478,8 @@ data. Bincode is the default production format; JSON is selected with - automatic reconnect with exponential backoff, peer health tracking and peer rotation *(Prototype)*; - periodic anti-entropy after missed pushes/reconnects *(Prototype)*; - bounded OpId deduplication and persisted dedup metadata *(Prototype)*; -- peer-to-peer gossip with K-fanout: architecture defined, full dynamic discovery/fanout remains future work *(Prototype)*. +- static, bootstrap, mDNS, DNS-SRV and file discovery through CLI, environment and TOML configuration *(Implemented)*; +- peer-to-peer gossip with K-fanout: architecture defined, SWIM membership and K-fanout dissemination remain future work *(Prototype)*. ### 5.5 Channel security *(Implemented)* @@ -876,7 +880,7 @@ flamegraphs with `pprof-rs` and load-phase heap profiles with `dhat`. ## 8. Use Cases -The use cases below are **concretely achievable today** with the primitives of `v0.1.4`. They do not describe visions: they describe what the runtime already knows how to do with the current stable feature set. +The use cases below are **concretely achievable today** with the primitives of `v0.1.5`. They do not describe visions: they describe what the runtime already knows how to do with the current stable feature set. ### 8.1 Distributed counters and metrics (example: `distributed_counter`) @@ -906,7 +910,7 @@ The compute is portable across Numax nodes: the same `.wasm` module can run on a **Problem.** Applications that must work without a connection (collaborative notes, distributed configurations, field applications, maritime/aerial/rural devices) and reconcile when they come back online, without imposing manual conflict resolution. -**Why Numax.** This is exactly the sweet spot of CRDTs: each node operates locally on its own store, changes propagate opportunistically, convergence is mathematically guaranteed. With PNCounter, LWW-Register, ORSet, LWW-Map and RGA available since `v0.1.0` and retained in `v0.1.4`, the model covers counters, statuses, observed-remove sets, replicated settings and ordered collaborative sequences. +**Why Numax.** This is exactly the sweet spot of CRDTs: each node operates locally on its own store, changes propagate opportunistically, convergence is mathematically guaranteed. With PNCounter, LWW-Register, ORSet, LWW-Map and RGA available since `v0.1.0` and retained in `v0.1.5`, the model covers counters, statuses, observed-remove sets, replicated settings and ordered collaborative sequences. The `distributed_chat` example (today in local-only mode) represents the skeleton of this use case. @@ -942,19 +946,28 @@ Numax is not AI. It is one of the things that AI can, comfortably, run on top of ## 10. Limitations -`v0.1.4` is the current stable release, building on the first stable `v0.1.0` line. We recognize its limits explicitly: +`v0.1.5` is the current stable release, building on the first stable `v0.1.0` line. We recognize its limits explicitly: -- **Network resilience is still prototype-grade.** Automatic reconnect, peer health tracking, peer rotation, anti-entropy and bounded dedup are implemented for configured peers, but full dynamic discovery and K-fanout gossip remain future work. +- **Network resilience is still prototype-grade.** `v0.1.5` combines automatic reconnect, peer health tracking, peer rotation, anti-entropy and bounded dedup with static, bootstrap, mDNS, DNS-SRV and file discovery. SWIM membership and K-fanout gossip remain future work. Local startup readiness is not peer or CRDT convergence. Recovery depends on retained operation and deduplication history, not merely on rediscovery; unrestricted lossless recovery is not guaranteed. - **Deduplication is bounded.** Recent duplicate remote operations are prevented across restart, but this is not an infinite causal history. Stronger guarantees would require a fuller durable op-log/causal metadata strategy. - **TLS/mTLS is implemented, but not yet hardened for all scenarios.** It is solid enough for controlled scenarios (dev, lab, defined deployments); the full hardening (rotation, advanced pinning, extreme hostile scenarios) continues. - **Observability is operational but intentionally small.** Structured logs, Prometheus-compatible metrics, health checks, a ready-made Prometheus/Grafana stack, a Grafana dashboard and PromQL examples are available. Deeper tracing and richer built-in dashboards remain future work. -- **Wire format and Host API are versioned but still young.** The current wire protocol is versioned (`PROTOCOL_VERSION = 4`) and supports bincode by default with JSON debug mode. Future incompatible changes must be explicit and versioned. +- **Wire format and Host API are versioned but still young.** `v0.1.5` uses wire protocol version `5` for bootstrap and rejects version `4` peers from `v0.1.4`. Both support Bincode (implemented with `wincode`) and JSON. This wire change does not change the persisted schema or guest ABI. - **Available CRDTs are still expanding.** GCounter, PNCounter, LWW-Register, ORSet, LWW-Map and RGA are implemented; additional CRDT families remain future work. - **It does not replace complex orchestrators.** It is not designed to manage extensive clusters or highly scalable deployments with advanced scheduling. - **Not optimized for CPU-bound workloads.** The focus is I/O and coordination, not intensive computation. - **Data models must be compatible with CRDTs.** Patterns based on locks or strong distributed transactions do not map directly. -These limits are not hidden weaknesses: they are the **honest perimeter** of the 0.1.4 release, which is useful today and still explicit about what remains future work. +These limits are not hidden weaknesses: they are the **honest perimeter** of the 0.1.5 release, which is useful today and still explicit about what remains future work. + +For `v0.1.5`, the [discovery contract](/numax/design/discovery-contract/) +adds explicit freshness, lifecycle and resource boundaries: observations retain +their timestamps across cached snapshots; anti-entropy uses active connections +on a cadence independent of discovery churn; Numax bounds mDNS retained +contributions globally, but `mdns-sd 0.21` exposes no configurable internal +cache bound. Bounded unregister/shutdown acknowledgements do not guarantee LAN +receipt of goodbye packets. NAT/WAN remains an open decision, not an adopted +traversal design. --- @@ -969,11 +982,11 @@ Numax proposes a unified runtime that combines: The goal is not to replicate the existing ecosystem, but **to reduce the self-imposed complexity** that today dominates distributed systems development, while preserving control over the necessary complexity of one's own domain. -`v0.1.4` is the current stable Numax release. It retains the real, tested foundation established by `v0.1.0` and hardened in `v0.1.1` - WASM runtime, sled store, six CRDT families, async replication, TCP networking, TLS 1.3 + mTLS, extended host APIs, modular SyncManager, explicit wire/schema versioning, typed protocol errors and offline datastore migration - and retains opt-in task, CPU and heap profiling, WASM and sync metrics, a blocking performance-regression gate, signed release checksums, SBOMs and fuzzing. The 0.1.4 additions are `nx serve`, the authenticated Management API, persistent module registration, binary-safe datastore inspection and cancellable one-shot guest execution. +`v0.1.5` is the current stable Numax release. It retains the real, tested foundation established by `v0.1.0` and hardened in `v0.1.1` WASM runtime, sled store, six CRDT families, async replication, TCP networking, TLS 1.3 + mTLS, extended host APIs, modular SyncManager, explicit wire/schema versioning, typed protocol errors and offline datastore migration - and retains opt-in task, CPU and heap profiling, WASM and sync metrics, a blocking performance-regression gate, signed release checksums, SBOMs and fuzzing. The 0.1.4 additions are `nx serve`, the authenticated Management API, persistent module registration, binary-safe datastore inspection and cancellable one-shot guest execution. Version 0.1.5 adds five peer discovery providers, bounded bootstrap address exchange and wire protocol version 5, with explicit lifecycle, resource and recovery boundaries. What is still missing is declared explicitly and tracked in the roadmap. Subsequent iterations will refine details, practical examples, comparisons and experimental results. -**`v0.1.4` is the current stable release.** It is built on code, tests and documented limits rather than promises; `v0.1.0` remains the first stable line it evolved from. +**`v0.1.5` is the current stable release.** It is built on code, tests and documented limits rather than promises; `v0.1.0` remains the first stable line it evolved from. In closing, I love software and I love numax. diff --git a/examples/README.md b/examples/README.md index 2cc626d..5401e83 100644 --- a/examples/README.md +++ b/examples/README.md @@ -21,6 +21,7 @@ Examples that replicate state across Numax nodes using CRDTs and converge throug | Example | Description | | --- | --- | +| [`discovery_lan`](discovery_lan/README.md) | Three mDNS-discovered daemons without `--peer`: SDK CRDT writes via authenticated local HTTP, offline operations and recovery; separate same-host test and three-device LAN procedure. | | [`distributed_ants`](distributed_ants/README.md) | Distributed Ant Colony Optimization swarm: a shared pheromone trail (PNCounter grid) emerges from many independent nodes. | | [`distributed_magnets`](distributed_magnets/README.md) | Distributed Magnetic Optimization Algorithm swarm: particles publish their position (LWW-Register) and pull toward whichever anchor or peer has the most mass. | | [`distributed_counter`](distributed_counter/README.md) | Grow-only distributed counter (GCounter). | diff --git a/examples/crypto_hashing/Cargo.lock b/examples/crypto_hashing/Cargo.lock index 99cc01a..0117e65 100644 --- a/examples/crypto_hashing/Cargo.lock +++ b/examples/crypto_hashing/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/crypto_hashing/Cargo.toml b/examples/crypto_hashing/Cargo.toml index b36cb74..b5b3676 100644 --- a/examples/crypto_hashing/Cargo.toml +++ b/examples/crypto_hashing/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/discovery_lan/Cargo.lock b/examples/discovery_lan/Cargo.lock new file mode 100644 index 0000000..f325c2e --- /dev/null +++ b/examples/discovery_lan/Cargo.lock @@ -0,0 +1,14 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "discovery_lan" +version = "0.1.5" +dependencies = [ + "nx-sdk", +] + +[[package]] +name = "nx-sdk" +version = "0.1.5" diff --git a/examples/discovery_lan/Cargo.toml b/examples/discovery_lan/Cargo.toml new file mode 100644 index 0000000..6bd6b39 --- /dev/null +++ b/examples/discovery_lan/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "discovery_lan" +version = "0.1.5" +edition = "2024" +publish = false + +[lib] +crate-type = ["cdylib"] + +[features] +increment = [] + +[dependencies] +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } + +[profile.release] +lto = true +opt-level = "z" +codegen-units = 1 +panic = "abort" + +[workspace] \ No newline at end of file diff --git a/examples/discovery_lan/README.md b/examples/discovery_lan/README.md new file mode 100644 index 0000000..17765bb --- /dev/null +++ b/examples/discovery_lan/README.md @@ -0,0 +1,368 @@ +# mDNS LAN discovery, CRDT replication and restart recovery + +Three `nx serve` daemons discover each other through real mDNS, **without any +`--peer`, static peers, bootstrap seed, or management peer injection**. HTTP +management registers and runs real WebAssembly guests using `nx-sdk`. + +- Writer build: increments `discovery-lan:visits` once through the GCounter SDK. +- Reader build: reads that counter and the local NodeId without creating CRDT + operations. It writes a **local-only observation** (`NodeId\nvalue`) under the + ordinary KV key `discovery-lan`. HTTP reads this observation; ordinary KV writes + are **not** the replication mechanism. Reserved `__nx/` data is never exposed. +- The test checks initial convergence, stopped-node absence, writes while stopped, + same-datastore restart, durable identity/local snapshot, missed-op recovery and + a new write from the restarted node. + +## Prerequisites and boundaries + +Rust with `wasm32-unknown-unknown`; Node.js 20+ for the device script; macOS or +Linux with usable IPv4 multicast. Build on each device (or transfer the two WASM +artifacts and a matching native `nx` executable yourself). No installation, +publication, remote command execution or external-resource deletion is performed +by the demo script. + +Use a **trusted isolated LAN**: mDNS announcements and the default TCP replication +transport are not authenticated/encrypted. The cluster label isolates discovery, +**not authorization**. This is not an mTLS demonstration. Do not use confidential +data; use the separate TLS example for certificate provisioning. Management is +authenticated using a random per-node token file, binds only to `127.0.0.1`, and +never requires `allow_non_loopback` or an insecure external HTTP endpoint. + +Allow UDP multicast 5353 and the selected TCP replication port between devices. +Wi-Fi client isolation, VLAN boundaries, VPN routing and firewalls may prevent +discovery/replication. Advertise the real local LAN IPv4, not loopback or `0.0.0.0`. +No NAT/WAN, routed multicast, device power loss, or recovery beyond retention is +claimed here. + +## Understanding `--listen` vs `--advertised-endpoint` + +In static clustering (`v0.1.4`), every node had to know all other nodes' IP addresses in advance via repeated `--peer` flags ($O(N^2)$ configuration). + +With discovery in `v0.1.5`+, each node only describes **itself** ($O(1)$ configuration per node): + +- `--listen `: The local socket address the daemon binds to (where the OS listens for incoming TCP connections). +- `--advertised-endpoint `: The address published over mDNS/Gossip for remote peers to dial back via TCP. + +### Why is `--advertised-endpoint` needed? +1. **Wildcard binding (`0.0.0.0`)**: If a node listens on `0.0.0.0:7000`, remote peers cannot dial `0.0.0.0`. The node must advertise its reachable unicast IP (e.g. `192.168.1.20:7000`). +2. **Multiple network interfaces**: When Wi-Fi, Ethernet, Docker, or VPN interfaces coexist, advertising explicitly prevents publishing an unreachable local interface. +3. **Multiple nodes on the same host**: When running several daemons locally on `127.0.0.1`, each daemon binds and advertises a distinct port (e.g. `127.0.0.1:7001`, `127.0.0.1:7002`), allowing automatic discovery without collisions. + +*Note: If `--listen` binds directly to a specific concrete IP (such as `192.168.1.20:7000`), Numax automatically derives the advertised endpoint if omitted.* + +## Execute in 5 minutes + +The [build](#build-repository-root) must already be complete on all three +devices. Use the same source revision and `CLUSTER`. Replace the example IPs. + +On A: + +```sh +export STATE="$HOME/numax-lan-a-015" +export LAN_IP="192.168.1.20" +export CLUSTER="numax-release-015-unique" +export INSTANCE="device-a" +``` + +On B: + +```sh +export STATE="$HOME/numax-lan-b-015" +export LAN_IP="192.168.1.21" +export CLUSTER="numax-release-015-unique" +export INSTANCE="device-b" +``` + +On C: + +```sh +export STATE="$HOME/numax-lan-c-015" +export LAN_IP="192.168.1.22" +export CLUSTER="numax-release-015-unique" +export INSTANCE="device-c" +``` + +On A, B and C: + +```sh +node examples/discovery_lan/demo.mjs init \ + --state "$STATE" \ + --lan-ip "$LAN_IP" \ + --cluster "$CLUSTER" \ + --instance "$INSTANCE" +``` + +Start a daemon on each device and leave it running: + +```sh +node examples/discovery_lan/demo.mjs start \ + --state "$STATE" \ + --nx "$PWD/target/release/nx" +``` + +Open a second terminal on each device, export its `STATE` again, then run: + +```sh +node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 2 --value 0 +``` + +On A, B and C, increment once: + +```sh +node examples/discovery_lan/demo.mjs increment --state "$STATE" +``` + +After all three increments, on A, B and C: + +```sh +node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 2 --value 3 +``` + +Stop C with Ctrl-C. On A and B, wait for its removal: + +```sh +node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 1 --value 3 +``` + +After both waits complete, increment once on A and B: + +```sh +node examples/discovery_lan/demo.mjs increment --state "$STATE" +``` + +Then on A and B: + +```sh +node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 1 --value 5 +``` + +Restart C with the same `STATE`: + +```sh +node examples/discovery_lan/demo.mjs start \ + --state "$STATE" \ + --nx "$PWD/target/release/nx" +``` + +On A, B and C: + +```sh +node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 2 --value 5 +``` + +Increment once on C: + +```sh +node examples/discovery_lan/demo.mjs increment --state "$STATE" +``` + +Then on A, B and C: + +```sh +node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 2 --value 6 +``` + +Stop every daemon with Ctrl-C. The test passes if: + +- every node finds two peers without `--peer`; +- values reach `0`, `3`, `5` and `6`; +- C keeps the same NodeId after restart; +- C recovers the two offline writes; +- all three daemons exit cleanly. + +Keep the `wait` output. Do not publish token files or state directories. + +## Build (repository root) + +```sh +rustup target add wasm32-unknown-unknown +cargo build -p nx-cli +cargo build --release --target wasm32-unknown-unknown --manifest-path examples/discovery_lan/Cargo.toml --target-dir examples/discovery_lan/target/reader +cargo build --release --target wasm32-unknown-unknown --manifest-path examples/discovery_lan/Cargo.toml --target-dir examples/discovery_lan/target/writer --features increment +``` + +Keep separate target directories: otherwise the second build overwrites the +reader artifact. The test asserts the modules have different content IDs. + +## Automated same-host E2E (also the CI invocation) + +Set `NUMAX_MDNS_LAN_IP` to an IPv4 address actually assigned to the host's LAN +interface. For example on macOS, find the active device with +`route -n get default`, then use `ipconfig getifaddr en0` (replace `en0` with that +device). A runner without a usable multicast interface must report the job as +unavailable, **not silently pass**. + +```sh +NUMAX_MDNS_E2E=1 NUMAX_MDNS_LAN_IP=192.168.1.20 cargo test -p nx-cli --test multiprocess_smoke discovery_lan::mdns_three_daemons_recover_missed_crdt_ops_after_restart -- --ignored --exact --nocapture --test-threads=1 +``` + +Replace the example IP. The test is both `#[ignore]` and explicitly environment +gated; running it explicitly without its prerequisites **fails**. The ordinary +workspace suite does not execute it. CI builds both guests and explicitly runs +this test on macOS, deriving the advertised IPv4 from the current default LAN +interface. An address from an earlier run may no longer belong to that interface. + +The test starts three **processes on one host**, binds TCP to a real LAN interface, +and exercises real mDNS multicast. It is **not proof of three-machine discovery**. +It uses an exclusive temporary directory, unique cluster/instances, independently +generated tokens, ephemeral port reservations, bounded condition polling and +process guards. Every assertion failure kills/reaps owned daemons and removes +only that test's directory. Normal completion checks graceful SIGTERM shutdown. +Failure diagnostics redact tokens. The test ignores inherited `NX_*` variables +so local settings cannot inject peers or weaken management authentication. + +## Three actual devices: one foreground daemon per device + +Use the same fresh cluster label on A/B/C, a different instance name on each, +and each device's own LAN IPv4. The following uses documentation/example values; +substitute addresses and an unused cluster name. Run from each checkout root. +`$HOME` already exists; each state directory must **not** exist before `init`. + +On **device A**: + +```sh +node examples/discovery_lan/demo.mjs init --state "$HOME/numax-lan-a" --lan-ip 192.168.1.20 --cluster lan-demo-unique --instance device-a +node examples/discovery_lan/demo.mjs start --state "$HOME/numax-lan-a" +``` + +On **device B**: + +```sh +node examples/discovery_lan/demo.mjs init --state "$HOME/numax-lan-b" --lan-ip 192.168.1.21 --cluster lan-demo-unique --instance device-b +node examples/discovery_lan/demo.mjs start --state "$HOME/numax-lan-b" +``` + +On **device C**: + +```sh +node examples/discovery_lan/demo.mjs init --state "$HOME/numax-lan-c" --lan-ip 192.168.1.22 --cluster lan-demo-unique --instance device-c +node examples/discovery_lan/demo.mjs start --state "$HOME/numax-lan-c" +``` + +The defaults are TCP replication 9000 and local management 9102; optional +`--network-port` and `--management-port` are accepted by `init`. If placing more +than one daemon on a single host, assign distinct ports and state directories; +that remains a **same-host** experiment. `start --nx /absolute/path/to/nx` selects +another native executable. Inherited `NX_*` overrides are removed on launch. + +Keep `start` running in the foreground. In a **second local terminal on each +device**, set `STATE` to its directory and run: + +```sh +STATE="$HOME/numax-lan-a" # use numax-lan-b or numax-lan-c on B/C +node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 2 --value 0 +``` + +Record each `node_id` and verify each device's `peer_ids` are exactly the other +two recorded identities. `connections` may contain inbound and outbound links +to the same identity; `--peers` counts **unique identities**, not TCP connections. + +### Reproducible offline/restart scenario + +Use fresh datastores and perform each increment exactly once. Wait commands poll +conditions with a default 60-second bound (`--timeout` allows 1–600 seconds); +there are no fixed startup or settling sleeps. + +1. **On A, B and C**, run one increment, then wait on all devices for value 3: + + ```sh + node examples/discovery_lan/demo.mjs increment --state "$STATE" + node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 2 --value 3 + ``` + + Run all three increments before expecting any wait for 3 to complete. + +2. **On C**, press Ctrl-C in its foreground `start` terminal. Wait for that + command to exit; do not reinitialize or remove its state directory. **On A + and B**, observe disconnection, then increment each once: + + ```sh + node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 1 --value 3 + node examples/discovery_lan/demo.mjs increment --state "$STATE" + node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 1 --value 5 + ``` + + Complete both disconnection waits before either increment, and both increments + before the waits for 5. These two operations occur while C has no running process. + +3. **On C**, restart with the exact same state directory: + + ```sh + node examples/discovery_lan/demo.mjs start --state "$HOME/numax-lan-c" + ``` + + **On all three devices**, wait for two identities and value 5: + + ```sh + node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 2 --value 5 + ``` + + Verify the recorded IDs are unchanged. The script also checks each local ID + against its exclusively created identity record. C has recovered the two + missed operations; the reader does not increment to manufacture convergence. + +4. **Only on C**, increment once. **On all devices**, wait for value 6: + + ```sh + # C only: + node examples/discovery_lan/demo.mjs increment --state "$STATE" + # All three: + node examples/discovery_lan/demo.mjs wait --state "$STATE" --peers 2 --value 6 + ``` + +5. Stop all foreground daemons with Ctrl-C and wait for clean exit. Datastores, + private logs, configuration, identity records and token files are deliberately + preserved; the script never removes external directories or kills processes + it did not spawn. Initialization refuses to overwrite an existing directory. + Startup failures and signals terminate/reap the owned child, escalating to + SIGKILL after a bounded 15-second graceful-shutdown attempt. + +`increment` is never automatically retried: a lost HTTP response can have an +ambiguous outcome. Inspect with `status` before deciding what to do. `status` +refreshes the reader projection without adding a CRDT operation: + +```sh +node examples/discovery_lan/demo.mjs status --state "$STATE" +``` + +## Retention and evidence + +Both demo and test explicitly configure `op_log_limit = 128` and +`seen_ops_limit = 128`, with `queued_ops_limit = 128` and anti-entropy every +200 ms. **Retention is count-based, not β€œ128 seconds”**. This fresh-cluster +scenario produces six CRDT operations total, only two during C's downtime. +Reading/status polling creates local KV observations but no CRDT operations. +It therefore remains below both retention bounds. Unrelated writers sharing a +cluster/datastore or repeated manual runs can invalidate that guarantee. +Do not infer that arbitrary downtime or an evicted operation will recover. + +For a release evidence record, retain command exit statuses and the identity/value +outputs at 0, 3, 5 and 6 from **each actual device**, plus platform, interface and +network topology. Do not attach token files. A passing local multiprocess test +is useful automated coverage, but does not substitute for this cross-device run. + +The automated test uses 60-second phase deadlines and 15-second shutdown +deadlines. It verifies unauthenticated management requests receive 401 and C's +previous local KV snapshot remains `(original NodeId, 3)` before running the +reader after restart. Recovery then comes from the CRDT path, not the snapshot. + +### Local verification β€” 2026-09-14 + +On the working tree based on `1674d5ee` (with the release-preparation changes), +the explicitly selected three-daemon E2E passed on macOS over the host's real +LAN interface: `0 -> 3 -> offline writes -> 5 -> restart recovery -> 6`. +The two-daemon mDNS discovery/removal test and the opt-in script lifecycle test +also passed. The latter verified a real SDK write and durable restart. + +These are **same-host** results. No three-device LAN run or remote CI matrix is +attested here; publication remains subject to those separate checks. Tokens and +private node directories are not release evidence and must not be published. + +## Script checks + +```sh +node --check examples/discovery_lan/demo.mjs +node --test examples/discovery_lan/demo.test.mjs +# Optional real single-daemon script lifecycle test, after building both guests: +NUMAX_DEMO_E2E=1 node --test examples/discovery_lan/demo.test.mjs +``` diff --git a/examples/discovery_lan/demo.mjs b/examples/discovery_lan/demo.mjs new file mode 100644 index 0000000..3cbf87e --- /dev/null +++ b/examples/discovery_lan/demo.mjs @@ -0,0 +1,240 @@ +#!/usr/bin/env node +// No dependencies, remote management, shell evaluation, or automatic deletion. +import { randomBytes } from 'node:crypto'; +import { spawn } from 'node:child_process'; +import { open, mkdir, readFile, writeFile } from 'node:fs/promises'; +import { networkInterfaces } from 'node:os'; +import { resolve, dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { parseArgs } from 'node:util'; +import { setTimeout as pollDelay } from 'node:timers/promises'; + +const here = dirname(fileURLToPath(import.meta.url)); +const SNAPSHOT = Buffer.from('discovery-lan').toString('base64url'); +const RETENTION = 128; // Operation count, NOT a time window. +const { values: options, positionals } = parseArgs({ + allowPositionals: true, + options: Object.fromEntries([ + 'state', 'lan-ip', 'cluster', 'instance', 'network-port', 'management-port', + 'nx', 'value', 'peers', 'timeout', + ].map(name => [name, { type: 'string' }])), +}); + +function check(condition, message) { + if (!condition) throw new Error(message); +} + +function numberOption(name, fallback, max) { + const text = options[name] ?? String(fallback); + check(/^\d+$/.test(text), `--${name} must be an integer`); + const value = Number(text); + check(Number.isSafeInteger(value) && value >= 1 && value <= max, `invalid --${name}`); + return value; +} + +async function initialize(state) { + for (const name of ['lan-ip', 'cluster', 'instance']) { + check(options[name], `init requires --${name}`); + } + const ip = options['lan-ip']; + const local = Object.values(networkInterfaces()).flat().some( + address => address?.family === 'IPv4' && !address.internal && address.address === ip, + ); + check(local, '--lan-ip must be an actual non-loopback local IPv4 interface'); + for (const name of ['cluster', 'instance']) { + check(/^[a-zA-Z0-9-]{1,50}$/.test(options[name]), `--${name}: use 1–50 letters, digits or hyphens`); + } + const network = numberOption('network-port', 9000, 65535); + const management = numberOption('management-port', 9102, 65535); + check(network !== management, 'network and management ports must differ'); + // Exclusive creation prevents overwriting an existing datastore/configuration. + // The parent directory must exist. Partial initialization is preserved on errors. + await mkdir(state, { mode: 0o700 }); + const token = randomBytes(32).toString('hex'); + await writeFile(join(state, 'management.token'), `${token}\n`, { flag: 'wx', mode: 0o600 }); + const q = JSON.stringify; + const config = `[network] +listen = ${q(`${ip}:${network}`)} +peers = [] + +[storage] +datastore_path = ${q(join(state, 'data'))} + +[management] +listen = "127.0.0.1:${management}" +token_file = ${q(join(state, 'management.token'))} +allow_non_loopback = false + +[discovery] +mode = "mdns" +cluster_id = ${q(options.cluster)} +instance_name = ${q(options.instance)} +advertised_endpoint = ${q(`${ip}:${network}`)} +max_candidates = 8 +max_instances = 8 + +[limits] +max_peers = 4 +queued_ops_limit = 128 +op_log_limit = ${RETENTION} +seen_ops_limit = ${RETENTION} +anti_entropy_interval = "200ms" +reconnect_initial_delay = "100ms" +reconnect_max_delay = "1s" +`; + await writeFile(join(state, 'node.toml'), config, { flag: 'wx', mode: 0o600 }); + await writeFile(join(state, 'control.json'), JSON.stringify({ management }), { flag: 'wx', mode: 0o600 }); + console.log(`Initialized ${state}; management stays on loopback; retention=${RETENTION} operations.`); +} + +async function client(state) { + const { management } = JSON.parse(await readFile(join(state, 'control.json'), 'utf8')); + check(Number.isInteger(management) && management > 0 && management <= 65535, 'invalid management port'); + const token = (await readFile(join(state, 'management.token'), 'utf8')).trim(); + check(/^[0-9a-f]{64}$/.test(token), 'invalid token file'); + return async function request(path, { method = 'GET', body, type, allowed = [200] } = {}) { + const response = await fetch(`http://127.0.0.1:${management}/api/v1/${path}`, { + method, body, redirect: 'error', signal: AbortSignal.timeout(5000), + headers: { Authorization: `Bearer ${token}`, ...(type ? { 'Content-Type': type } : {}) }, + }); + // Do not echo headers, tokens, or arbitrary response bodies in errors. + check(allowed.includes(response.status), `${method} ${path}: HTTP ${response.status}`); + return response; + }; +} + +async function register(request, mode) { + const wasm = await readFile(join(here, 'target', mode, 'wasm32-unknown-unknown', 'release', 'discovery_lan.wasm')); + const response = await request('modules', { method: 'POST', body: wasm, type: 'application/wasm', allowed: [200, 201] }); + const { id } = await response.json(); + check(/^[0-9a-f]{64}$/.test(id), 'invalid module id'); + return id; +} + +async function run(request, module) { + await request(`modules/${module}/runs`, { method: 'POST', allowed: [204] }); +} + +async function snapshot(request, reader, state) { + await run(request, reader); // Read-only CRDT operation; refreshes LOCAL KV projection. + const response = await request(`keys/${SNAPSHOT}`); + const [nodeId, value, extra] = (await response.text()).split('\n'); + check(nodeId && /^\d+$/.test(value) && extra === undefined, 'invalid guest snapshot'); + const identityPath = join(state, 'identity'); + try { + await writeFile(identityPath, nodeId, { flag: 'wx', mode: 0o600 }); + } catch (error) { + if (error.code !== 'EEXIST') throw error; + check(await readFile(identityPath, 'utf8') === nodeId, 'node identity changed; do not replace the datastore'); + } + const peers = await (await request('peers?limit=10')).json(); + check(peers.next_cursor === null && Array.isArray(peers.items), 'unexpected peer page'); + const ids = peers.items.map(peer => peer.node_id); + check(!ids.includes(nodeId), 'unexpected self connection'); + // Management lists connections; inbound/outbound links may share an identity. + return { node_id: nodeId, value, peer_ids: [...new Set(ids)].sort(), connections: peers.items }; +} + +async function waitFor(label, timeout, condition, isAlive = () => true) { + const deadline = performance.now() + timeout; + let last = 'condition not met'; + do { + check(isAlive(), `${label}: daemon exited`); + try { + const result = await condition(); + if (result) return result; + } catch (error) { + last = error.message; + } + if (performance.now() >= deadline) break; + await pollDelay(200); // Condition polling only; no fixed startup/settling sleep. + } while (performance.now() < deadline); + throw new Error(`${label}: timeout (${last})`); +} + +async function start(state, timeout) { + const request = await client(state); + // Check configuration exists before spawning. nx remains the configuration validator. + await readFile(join(state, 'node.toml')); + const log = await open(join(state, 'daemon.log'), 'a', 0o600); + const env = Object.fromEntries(Object.entries(process.env).filter(([name]) => !name.startsWith('NX_'))); + let child; + let ended = false; + let exit; + try { + child = spawn(resolve(options.nx ?? join(here, '..', '..', 'target', 'debug', 'nx')), + ['serve', '--config', join(state, 'node.toml')], + { env, stdio: ['ignore', log.fd, log.fd] }); + // Attach before the first await: spawn errors can arrive on the next tick. + exit = new Promise(resolveExit => { + child.once('error', () => { ended = true; resolveExit({ error: 'cannot start nx; check --nx' }); }); + child.once('exit', (code, signal) => { ended = true; resolveExit({ code, signal }); }); + }); + } finally { + await log.close(); + } + let stopping; + function stop() { + stopping ??= (async () => { + if (ended) return; + child.kill('SIGTERM'); + const escalation = setTimeout(() => child.kill('SIGKILL'), 15000); + try { await exit; } finally { clearTimeout(escalation); } + })(); + return stopping; + } + const signal = () => { void stop(); }; + process.on('SIGINT', signal); + process.on('SIGTERM', signal); + try { + await waitFor('daemon readiness', timeout, async () => { + await request('ready'); + return true; + }, () => !ended); + check(!ended, 'daemon exited during readiness'); + console.log(`Daemon ready. Use another local terminal for status/increment/wait. Ctrl-C stops it; ${state} is preserved.`); + const result = await exit; + check(result.code === 0, result.error ?? `daemon exited (code=${result.code}, signal=${result.signal}); inspect private daemon.log`); + } finally { + await stop(); + process.off('SIGINT', signal); + process.off('SIGTERM', signal); + } +} + +async function main() { + check(positionals.length === 1 && ['init', 'start', 'increment', 'status', 'wait'].includes(positionals[0]), + 'Usage: node demo.mjs init|start|increment|status|wait --state PATH (see README)'); + check(options.state, '--state is required'); + const state = resolve(options.state); + const action = positionals[0]; + const timeout = numberOption('timeout', 60, 600) * 1000; + if (action === 'init') return initialize(state); + if (action === 'start') return start(state, timeout); + if (action === 'wait') { + check(options.value !== undefined || options.peers !== undefined, 'wait requires --value and/or --peers'); + if (options.value !== undefined) check(/^\d+$/.test(options.value), '--value must be a nonnegative integer'); + if (options.peers !== undefined) check(/^[0-2]$/.test(options.peers), '--peers must be 0, 1 or 2'); + } + const request = await client(state); + if (action === 'increment') { + // Never retry a mutation: a lost HTTP response has an ambiguous outcome. + await run(request, await register(request, 'writer')); + } + const reader = await register(request, 'reader'); + const observe = () => snapshot(request, reader, state); + const result = action === 'wait' + ? await waitFor('convergence', timeout, async () => { + const current = await observe(); + const valueMatches = options.value === undefined || BigInt(current.value) === BigInt(options.value); + const peersMatch = options.peers === undefined || current.peer_ids.length === Number(options.peers); + return valueMatches && peersMatch ? current : false; + }) + : await observe(); + console.log(JSON.stringify(result, null, 2)); +} + +main().catch(error => { + console.error(`discovery_lan: ${error.message}`); + process.exitCode = 1; +}); \ No newline at end of file diff --git a/examples/discovery_lan/demo.test.mjs b/examples/discovery_lan/demo.test.mjs new file mode 100644 index 0000000..b5a47ab --- /dev/null +++ b/examples/discovery_lan/demo.test.mjs @@ -0,0 +1,128 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { once } from 'node:events'; +import { createServer } from 'node:net'; +import { mkdtempSync, readFileSync, rmSync, statSync } from 'node:fs'; +import { networkInterfaces, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const script = fileURLToPath(new URL('./demo.mjs', import.meta.url)); +const execute = (...args) => spawnSync(process.execPath, [script, ...args], { encoding: 'utf8', timeout: 10000 }); + +test('rejects missing arguments and loopback advertisement', () => { + assert.notEqual(execute().status, 0); + const result = execute('init', '--state', join(tmpdir(), 'unused-numax-demo'), '--lan-ip', '127.0.0.1', '--cluster', 'test', '--instance', 'a'); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /non-loopback local IPv4/); +}); + +const lan = Object.values(networkInterfaces()).flat().find(address => address?.family === 'IPv4' && !address.internal)?.address; +test('creates private loopback management config and refuses to overwrite it', () => { + // Initialization validates a real interface, but this test runs no daemon. + assert.ok(lan, 'configuration tests require a non-loopback local IPv4 interface'); + const root = mkdtempSync(join(tmpdir(), 'numax-demo-test-')); + try { + const state = join(root, 'node'); + const args = ['init', '--state', state, '--lan-ip', lan, '--cluster', 'test-private', '--instance', 'a']; + const first = execute(...args); + assert.equal(first.status, 0, first.stderr); + const token = readFileSync(join(state, 'management.token'), 'utf8').trim(); + assert.match(token, /^[0-9a-f]{64}$/); + assert.ok(!first.stdout.includes(token) && !first.stderr.includes(token)); + const config = readFileSync(join(state, 'node.toml'), 'utf8'); + assert.match(config, /listen = "127\.0\.0\.1:9102"/); + assert.match(config, /allow_non_loopback = false/); + assert.match(config, /peers = \[\]/); + assert.match(config, /op_log_limit = 128/); + assert.match(config, /seen_ops_limit = 128/); + assert.ok(!config.includes(token)); + if (process.platform !== 'win32') { + assert.equal(statSync(state).mode & 0o777, 0o700); + assert.equal(statSync(join(state, 'management.token')).mode & 0o777, 0o600); + } + assert.notEqual(execute(...args).status, 0); + assert.equal(readFileSync(join(state, 'management.token'), 'utf8').trim(), token); + const failure = execute('start', '--state', state, '--nx', join(root, 'missing-nx'), '--timeout', '1'); + assert.notEqual(failure.status, 0); + assert.ok(!failure.stderr.includes(token)); + } finally { + // Only resources exclusively created by this test are removed. + rmSync(root, { recursive: true, force: true }); + } +}); + +test('real script lifecycle: SDK write, HTTP observation, stop and durable restart', { + skip: process.env.NUMAX_DEMO_E2E !== '1', timeout: 90000, +}, async () => { + assert.ok(lan, 'a real LAN interface is required'); + const root = mkdtempSync(join(tmpdir(), 'numax-demo-live-')); + const state = join(root, 'node'); + const network = createServer(); + const management = createServer(); + let child; + let childExit; + let output = ''; + async function stop() { + if (!child) return; + child.kill('SIGTERM'); + const timer = setTimeout(() => child.kill('SIGKILL'), 20000); + try { + const [code, signal] = await childExit; + assert.equal(signal, null, output); + assert.equal(code, 0, output); + } finally { + clearTimeout(timer); + child = undefined; + } + } + function start() { + output = ''; + child = spawn(process.execPath, [script, 'start', '--state', state], { stdio: ['ignore', 'pipe', 'pipe'] }); + child.stdout.on('data', data => { output += data; }); + child.stderr.on('data', data => { output += data; }); + childExit = once(child, 'exit'); + } + function command(...args) { + const result = execute(...args, '--state', state); + assert.equal(result.status, 0, `${result.stderr}\n${output}`); + return result.stdout; + } + try { + network.listen(0, lan); + await once(network, 'listening'); + management.listen(0, '127.0.0.1'); + await once(management, 'listening'); + command('init', '--lan-ip', lan, '--cluster', `script-${process.pid}-${Date.now()}`, + '--instance', 'script-node', '--network-port', String(network.address().port), + '--management-port', String(management.address().port)); + await Promise.all([new Promise(resolve => network.close(resolve)), new Promise(resolve => management.close(resolve))]); + start(); + // Observe readiness in the wrapper output; no fixed startup delay. + async function ready() { + const deadline = Date.now() + 15000; + while (!output.includes('Daemon ready.')) { + assert.equal(child.exitCode, null, output); + assert.ok(Date.now() < deadline, `script readiness timeout: ${output}`); + await new Promise(resolve => setTimeout(resolve, 50)); + } + } + await ready(); + const initial = JSON.parse(command('wait', '--peers', '0', '--value', '0')); + assert.equal(JSON.parse(command('increment')).value, '1'); + await stop(); + start(); + await ready(); + const recovered = JSON.parse(command('wait', '--peers', '0', '--value', '1')); + assert.equal(recovered.node_id, initial.node_id); + assert.equal(JSON.parse(command('increment')).value, '2'); + await stop(); + } finally { + try { await stop(); } finally { + network.close(); + management.close(); + rmSync(root, { recursive: true, force: true }); + } + } +}); \ No newline at end of file diff --git a/examples/discovery_lan/src/lib.rs b/examples/discovery_lan/src/lib.rs new file mode 100644 index 0000000..ee12d40 --- /dev/null +++ b/examples/discovery_lan/src/lib.rs @@ -0,0 +1,31 @@ +//! The default build observes CRDT state without creating replication operations. +//! The `increment` build adds exactly one before observing it. +//! HTTP can read the local snapshot without accessing the reserved CRDT namespace. + +use nx_sdk::{crdt::gcounter, db, net}; + +const COUNTER_KEY: &str = "discovery-lan:visits"; +const SNAPSHOT_KEY: &str = "discovery-lan"; + +fn execute() -> nx_sdk::Result<()> { + #[cfg(feature = "increment")] + gcounter::inc(COUNTER_KEY, 1)?; + + let node_id = net::node_id()?; + let value = gcounter::value(COUNTER_KEY)?; + // This KV entry is local observation only; it is NOT the replicated counter. + db::set(SNAPSHOT_KEY, format!("{node_id}\n{value}").as_bytes())?; + Ok(()) +} + +#[unsafe(no_mangle)] +pub extern "C" fn run() { + if execute().is_err() { + nx_sdk::log("discovery_lan: SDK operation failed"); + // A failed SDK operation must fail the HTTP run, not look successful. + #[cfg(target_arch = "wasm32")] + core::arch::wasm32::unreachable(); + #[cfg(not(target_arch = "wasm32"))] + panic!("discovery_lan is a WebAssembly guest"); + } +} diff --git a/examples/distributed_ants/Cargo.lock b/examples/distributed_ants/Cargo.lock index d95f7c0..8beda6a 100644 --- a/examples/distributed_ants/Cargo.lock +++ b/examples/distributed_ants/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/distributed_ants/Cargo.toml b/examples/distributed_ants/Cargo.toml index 9207bc1..276190f 100644 --- a/examples/distributed_ants/Cargo.toml +++ b/examples/distributed_ants/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/distributed_chat/Cargo.lock b/examples/distributed_chat/Cargo.lock index bbde441..e95229f 100644 --- a/examples/distributed_chat/Cargo.lock +++ b/examples/distributed_chat/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/distributed_chat/Cargo.toml b/examples/distributed_chat/Cargo.toml index 76c44e9..2f29a93 100644 --- a/examples/distributed_chat/Cargo.toml +++ b/examples/distributed_chat/Cargo.toml @@ -7,6 +7,6 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [workspace] diff --git a/examples/distributed_comments/Cargo.lock b/examples/distributed_comments/Cargo.lock index cadee7a..f44767f 100644 --- a/examples/distributed_comments/Cargo.lock +++ b/examples/distributed_comments/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/distributed_comments/Cargo.toml b/examples/distributed_comments/Cargo.toml index 1e52650..b47445c 100644 --- a/examples/distributed_comments/Cargo.toml +++ b/examples/distributed_comments/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/distributed_counter/Cargo.lock b/examples/distributed_counter/Cargo.lock index 9c9b92b..9845ca1 100644 --- a/examples/distributed_counter/Cargo.lock +++ b/examples/distributed_counter/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/distributed_counter/Cargo.toml b/examples/distributed_counter/Cargo.toml index c9b25b8..5ce0c33 100644 --- a/examples/distributed_counter/Cargo.toml +++ b/examples/distributed_counter/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/distributed_inventory/Cargo.lock b/examples/distributed_inventory/Cargo.lock index ed760ec..5ce16c8 100644 --- a/examples/distributed_inventory/Cargo.lock +++ b/examples/distributed_inventory/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/distributed_inventory/Cargo.toml b/examples/distributed_inventory/Cargo.toml index dcbe8f1..cd7499e 100644 --- a/examples/distributed_inventory/Cargo.toml +++ b/examples/distributed_inventory/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/distributed_magnets/Cargo.lock b/examples/distributed_magnets/Cargo.lock index 767b160..09ce7b5 100644 --- a/examples/distributed_magnets/Cargo.lock +++ b/examples/distributed_magnets/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/distributed_magnets/Cargo.toml b/examples/distributed_magnets/Cargo.toml index 3425a0c..b481cc0 100644 --- a/examples/distributed_magnets/Cargo.toml +++ b/examples/distributed_magnets/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/distributed_settings/Cargo.lock b/examples/distributed_settings/Cargo.lock index 4b5283b..b5f954f 100644 --- a/examples/distributed_settings/Cargo.lock +++ b/examples/distributed_settings/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/distributed_settings/Cargo.toml b/examples/distributed_settings/Cargo.toml index 9a014af..cf016e2 100644 --- a/examples/distributed_settings/Cargo.toml +++ b/examples/distributed_settings/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/distributed_status/Cargo.lock b/examples/distributed_status/Cargo.lock index 1361bab..65a7773 100644 --- a/examples/distributed_status/Cargo.lock +++ b/examples/distributed_status/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/distributed_status/Cargo.toml b/examples/distributed_status/Cargo.toml index c54d303..4de78eb 100644 --- a/examples/distributed_status/Cargo.toml +++ b/examples/distributed_status/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/distributed_tags/Cargo.lock b/examples/distributed_tags/Cargo.lock index 094866c..b66e7bd 100644 --- a/examples/distributed_tags/Cargo.lock +++ b/examples/distributed_tags/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/distributed_tags/Cargo.toml b/examples/distributed_tags/Cargo.toml index 908c42a..988ff9f 100644 --- a/examples/distributed_tags/Cargo.toml +++ b/examples/distributed_tags/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/hello_sdk/Cargo.lock b/examples/hello_sdk/Cargo.lock index bbad9a0..363cd40 100644 --- a/examples/hello_sdk/Cargo.lock +++ b/examples/hello_sdk/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/hello_sdk/Cargo.toml b/examples/hello_sdk/Cargo.toml index 436f45f..16a45a7 100644 --- a/examples/hello_sdk/Cargo.toml +++ b/examples/hello_sdk/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/kv_counter/Cargo.lock b/examples/kv_counter/Cargo.lock index 3a5540b..502ee48 100644 --- a/examples/kv_counter/Cargo.lock +++ b/examples/kv_counter/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/kv_counter/Cargo.toml b/examples/kv_counter/Cargo.toml index ee90bee..30e6080 100644 --- a/examples/kv_counter/Cargo.toml +++ b/examples/kv_counter/Cargo.toml @@ -7,6 +7,6 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [workspace] diff --git a/examples/kv_get_set_delete/Cargo.lock b/examples/kv_get_set_delete/Cargo.lock index c858a5e..3a89c58 100644 --- a/examples/kv_get_set_delete/Cargo.lock +++ b/examples/kv_get_set_delete/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/kv_get_set_delete/Cargo.toml b/examples/kv_get_set_delete/Cargo.toml index 99b58ce..ea120c4 100644 --- a/examples/kv_get_set_delete/Cargo.toml +++ b/examples/kv_get_set_delete/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/kv_sdk_roundtrip/Cargo.lock b/examples/kv_sdk_roundtrip/Cargo.lock index a495e65..8a6ffeb 100644 --- a/examples/kv_sdk_roundtrip/Cargo.lock +++ b/examples/kv_sdk_roundtrip/Cargo.lock @@ -11,4 +11,4 @@ dependencies = [ [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" diff --git a/examples/kv_sdk_roundtrip/Cargo.toml b/examples/kv_sdk_roundtrip/Cargo.toml index d82655f..d20f03b 100644 --- a/examples/kv_sdk_roundtrip/Cargo.toml +++ b/examples/kv_sdk_roundtrip/Cargo.toml @@ -7,6 +7,6 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [workspace] diff --git a/examples/time_clock/Cargo.lock b/examples/time_clock/Cargo.lock index 74e14e9..239ee50 100644 --- a/examples/time_clock/Cargo.lock +++ b/examples/time_clock/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" [[package]] name = "time_clock" diff --git a/examples/time_clock/Cargo.toml b/examples/time_clock/Cargo.toml index 650efd4..90884fd 100644 --- a/examples/time_clock/Cargo.toml +++ b/examples/time_clock/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/examples/vote_tally_tls/Cargo.lock b/examples/vote_tally_tls/Cargo.lock index fce2545..419c024 100644 --- a/examples/vote_tally_tls/Cargo.lock +++ b/examples/vote_tally_tls/Cargo.lock @@ -4,7 +4,7 @@ version = 4 [[package]] name = "nx-sdk" -version = "0.1.4" +version = "0.1.5" [[package]] name = "vote_tally_tls" diff --git a/examples/vote_tally_tls/Cargo.toml b/examples/vote_tally_tls/Cargo.toml index d5f1cf8..a879b1d 100644 --- a/examples/vote_tally_tls/Cargo.toml +++ b/examples/vote_tally_tls/Cargo.toml @@ -7,7 +7,7 @@ edition = "2024" crate-type = ["cdylib"] [dependencies] -nx-sdk = { version = "0.1.4", path = "../../crates/nx-sdk" } +nx-sdk = { version = "0.1.5", path = "../../crates/nx-sdk" } [profile.release] lto = true diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index f8013f6..a8ae43f 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -491,7 +491,7 @@ dependencies = [ [[package]] name = "nx-net" -version = "0.1.4" +version = "0.1.5" dependencies = [ "hex", "nx-sync", @@ -511,7 +511,7 @@ dependencies = [ [[package]] name = "nx-sync" -version = "0.1.4" +version = "0.1.5" dependencies = [ "serde", "serde_json",