From 596226362af38b8ff4f09207b9b2d4843598f0ea Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Fri, 4 Sep 2026 21:32:53 -0700 Subject: [PATCH 1/2] Take over the iroh-layer e2e tests from tunnel-rs with a minimal harness The relay failover suite and the auth/connectivity checks that tunnel-rs ran end to end exercise only this crate, so they now live here, against examples/e2e: a server that binds through the shared builder, runs the home-relay failover beside its accept loop, and answers the endpoint-bound auth transcript plus an echo; a client that dials through the configured relays and proves its key. The Python fixtures (fake relay, delay proxy) and key generation are harness subcommands, so the suites need only cargo and iroh-relay. CI runs the offline suites. ROADMAP.md records the idea of folding flexaccess-keys into this repository as a subcrate, without doing it. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0163AtLdT9AWeDE3EYmhFYXK --- .github/workflows/ci.yml | 33 +++ Cargo.lock | 265 ++++++++++++++++++- Cargo.toml | 5 + README.md | 20 ++ ROADMAP.md | 42 +++ e2e/README.md | 66 +++++ e2e/lib.sh | 205 +++++++++++++++ e2e/run_e2e.sh | 243 ++++++++++++++++++ e2e/run_relay_failover.sh | 501 ++++++++++++++++++++++++++++++++++++ examples/e2e/main.rs | 523 ++++++++++++++++++++++++++++++++++++++ 10 files changed, 1890 insertions(+), 13 deletions(-) create mode 100644 ROADMAP.md create mode 100644 e2e/README.md create mode 100644 e2e/lib.sh create mode 100755 e2e/run_e2e.sh create mode 100755 e2e/run_relay_failover.sh create mode 100644 examples/e2e/main.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b6d3a33..8d2e9e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -54,3 +54,36 @@ jobs: run: cargo clippy --all-targets --all-features -- -D warnings - name: Test run: cargo test --all-features + + # The end-to-end suites that need no public infrastructure: the auth + # transcript and connectivity over two local iroh-relay instances, and the + # relay failover scenarios (which take ~6 minutes by design: a 60 s outage + # window and a 90 s restore probe). See e2e/README.md. + e2e: + name: e2e (local relays) + runs-on: ubuntu-latest + env: + IROH_RELAY_VERSION: "1.1.0" + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + - uses: Swatinem/rust-cache@v2 + with: + key: e2e + - name: Cache iroh-relay + id: relay-cache + uses: actions/cache@v4 + with: + path: ~/.cargo/bin/iroh-relay + key: iroh-relay-${{ runner.os }}-${{ env.IROH_RELAY_VERSION }} + - name: Install iroh-relay + if: steps.relay-cache.outputs.cache-hit != 'true' + run: cargo install iroh-relay --version "$IROH_RELAY_VERSION" --features server --locked + - name: Build the harness + run: cargo build --example e2e + - name: Auth + connectivity (custom relays) + run: ./e2e/run_e2e.sh --local-relays + - name: Auth + connectivity (relay-only) + run: ./e2e/run_e2e.sh --local-relays --relay-only + - name: Relay failover + run: ./e2e/run_relay_failover.sh diff --git a/Cargo.lock b/Cargo.lock index 5c05f62..3532ea2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -76,6 +76,56 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + +[[package]] +name = "anstyle-parse" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys 0.61.2", +] + [[package]] name = "anyhow" version = "1.0.104" @@ -172,6 +222,12 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.1" @@ -282,6 +338,46 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 3.0.4", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "cmov" version = "0.5.4" @@ -297,6 +393,12 @@ dependencies = [ "thiserror 2.0.20", ] +[[package]] +name = "colorchoice" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" + [[package]] name = "combine" version = "4.6.8" @@ -505,6 +607,37 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "defmt" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +dependencies = [ + "bitflags 1.3.2", + "defmt-macros", +] + +[[package]] +name = "defmt-macros" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +dependencies = [ + "defmt-parser", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "defmt-parser" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +dependencies = [ + "thiserror 2.0.20", +] + [[package]] name = "der" version = "0.8.1" @@ -567,7 +700,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38" dependencies = [ - "bitflags", + "bitflags 2.13.1", "block2", "libc", "objc2", @@ -651,6 +784,29 @@ dependencies = [ "syn 3.0.4", ] +[[package]] +name = "env_filter" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "900d271a03799a1ee8d1ca9b19893b48ca674a9284fefcfb85f05e74ed314217" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de671bd27a75a797dc9ae289ba1e77276e75e2026408aab65185384e2d5cd3f6" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -691,6 +847,8 @@ version = "0.0.7" dependencies = [ "anyhow", "base64", + "clap", + "env_logger", "flexaccess-keys", "futures", "iroh", @@ -1528,12 +1686,54 @@ dependencies = [ "ws_stream_wasm", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jiff" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +dependencies = [ + "defmt", + "jiff-core", + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +dependencies = [ + "defmt", +] + +[[package]] +name = "jiff-static" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +dependencies = [ + "jiff-core", + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "jni" version = "0.21.1" @@ -1860,7 +2060,7 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2288fcb784eb3defd5fb16f4c4160d5f477de192eac730f43e1d11c24d9a007" dependencies = [ - "bitflags", + "bitflags 2.13.1", "libc", "log", "netlink-packet-core 0.8.2", @@ -1872,7 +2072,7 @@ version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59d48ffc8b73a506e1ff0878528c72668f2bfbc3d875b6be890a96be5d0d5576" dependencies = [ - "bitflags", + "bitflags 2.13.1", "libc", "log", "netlink-packet-core 0.9.0", @@ -2079,7 +2279,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.13.1", "block2", "dispatch2", "libc", @@ -2092,7 +2292,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c71e34919aba0d701380d911702455038a8a3587467fe0141d6a71501e7ffe48" dependencies = [ - "bitflags", + "bitflags 2.13.1", "objc2", "objc2-core-foundation", "objc2-foundation", @@ -2112,7 +2312,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ - "bitflags", + "bitflags 2.13.1", "block2", "libc", "objc2", @@ -2125,7 +2325,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "709fe137109bd1e8b5a99390f77a7d8b2961dafc1a1c5db8f2e60329ad6d895a" dependencies = [ - "bitflags", + "bitflags 2.13.1", "objc2", "objc2-core-foundation", ] @@ -2146,7 +2346,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7216bd11cbda54ccabcab84d523dc93b858ec75ecfb3a7d89513fa22464da396" dependencies = [ - "bitflags", + "bitflags 2.13.1", "dispatch2", "libc", "objc2", @@ -2164,6 +2364,12 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "opaque-debug" version = "0.3.1" @@ -2316,6 +2522,15 @@ dependencies = [ "serde", ] +[[package]] +name = "portable-atomic-util" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" +dependencies = [ + "portable-atomic", +] + [[package]] name = "portmapper" version = "0.19.3" @@ -2469,7 +2684,19 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.13.1", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", ] [[package]] @@ -2684,7 +2911,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.13.1", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -2842,7 +3069,7 @@ version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b6f884fa9a8d48101774bfbd3aeb81e968dd22cffd19a372da69f183db22c1a" dependencies = [ - "bitflags", + "bitflags 2.13.1", ] [[package]] @@ -2912,6 +3139,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "strum" version = "0.28.0" @@ -3002,7 +3235,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.13.1", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -3267,7 +3500,7 @@ version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ - "bitflags", + "bitflags 2.13.1", "bytes", "futures-util", "http", @@ -3418,6 +3651,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.26.0" diff --git a/Cargo.toml b/Cargo.toml index 7e85634..0e0d9b8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,6 +34,11 @@ tokio = { version = "1", features = ["macros", "rt", "sync", "time"] } iroh-mdns-address-lookup = { version = "0.5", optional = true } [dev-dependencies] +# The e2e harness (`examples/e2e`, driven by the scripts in `e2e/`): a +# command line and a logger for a server/client pair that exercises this +# crate over real relays. Dev-only, so consumers never pay for them. +clap = { version = "4", features = ["derive", "env"] } +env_logger = "0.11" # Test double for iroh's `Watcher`-based status APIs (the relay failover tests # drive a plain `Watchable`); the same crate iroh itself re-exports `Watcher` from. n0-watcher = "1" diff --git a/README.md b/README.md index e8ff8b5..a2ef398 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,21 @@ cargo clippy --all-targets --all-features -- -D warnings cargo test --all-features ``` +### End-to-end tests + +The e2e tests for the shared iroh layer live in [`e2e/`](e2e/README.md), run +against real relays through a minimal harness (`examples/e2e`) rather than +through a product: `run_e2e.sh` covers the auth transcript and connectivity +(default relays, custom relays, relay-only), `run_relay_failover.sh` covers +the startup probe and the in-place home-relay failover against two local +`iroh-relay` instances, fully offline. CI runs the offline suites. + +```sh +cargo install iroh-relay --features server # one-time +./e2e/run_e2e.sh --local-relays --relay-only +./e2e/run_relay_failover.sh +``` + To iterate against a consumer before tagging, point the consumer at a local checkout without committing it: @@ -90,6 +105,11 @@ checkout without committing it: flexaccess-iroh = { path = "../flexaccess-iroh" } ``` +## Roadmap + +See [ROADMAP.md](ROADMAP.md) for what is under consideration, including +folding [flexaccess-keys] into this repository as a subcrate. + ## Releasing Bump `version` in `Cargo.toml`, merge, and run the **Release** workflow: it diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..2b8cd27 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,42 @@ +# Roadmap + +Things under consideration for this crate, none of them committed. Each entry +says what would change and what has to be true first. + +## Fold flexaccess-keys into this repository as a subcrate + +Today [flexaccess-keys] is its own repository, consumed here by git tag and +re-exported as `flexaccess_keys`; every product pins both tags and has to keep +them in step (this crate signs and verifies with exactly the version it +re-exports). The idea is to make this repository a Cargo workspace with +`flexaccess-keys` as a member crate next to `flexaccess-iroh`, so the key +format and the transcript over it are versioned, tagged, tested, and reviewed +together, and a product pins one tag for both. + +What would change: + +- A workspace root with two members; `flexaccess-iroh` depends on + `flexaccess-keys` by `path`. Consumers keep depending on each crate by git + tag (`package = "flexaccess-keys"` from the same repository), so the + `flexaccess-keys` CLI, the iOS/FFI consumers, and anything that wants the + key format without iroh keep working without pulling iroh in. +- One release workflow tagging the repository once; both crates carry the same + version. +- The e2e harness's `keygen` and the `flexaccess-keys` CLI would share the + same code path directly rather than via the re-export. + +What has to be settled first: + +- Whether every consumer of `flexaccess-keys` is happy to take it from this + repository's tags, or whether the standalone repository must live on (in + which case a subcrate here only creates a second source of truth and the + idea should be dropped). +- The CI and release workflows here assume a single package + (`cargo metadata … .packages[0]`, the minimum-iroh job's `cargo update -p + iroh`); they need to become workspace-aware. +- Whether `flexaccess-keys`' own release binaries (the CLI downloaded by + product e2e scripts) keep being published, and from which repository. + +Not started; nothing in this repository is arranged around it yet. + +[flexaccess-keys]: https://github.com/flexaccessdev/flexaccess-keys diff --git a/e2e/README.md b/e2e/README.md new file mode 100644 index 0000000..d7920d1 --- /dev/null +++ b/e2e/README.md @@ -0,0 +1,66 @@ +# flexaccess-iroh end-to-end tests + +The e2e tests for the shared iroh layer live here, with the crate they test, +so a change to the relay probe, the endpoint builder, the home-relay failover, +or the auth transcript is verified against real relays in the same PR. They +used to run against tunnel-rs; now a product's e2e suite only has to cover +what the product adds. + +## The harness + +`examples/e2e` is the smallest program that puts every module of this crate +through real relays — deliberately not an application. Built with +`cargo build --example e2e` (the scripts do it), it offers: + +| Subcommand | Role | +|---|---| +| `server` | binds an endpoint with the shared builder (`--relay-url`, `--relay-only`, an optional `E2E_SERVER_SECRET` identity), runs `relay_failover::fail_over_home_relay` beside its accept loop, and answers one request per connection: the endpoint-bound auth transcript from `flexaccess_iroh::auth` against `--authorized-keys`, then an echo of the client's message. Logs `EndpointId: …` and `Waiting for clients to connect` when ready. | +| `client` | builds an ephemeral endpoint the same way, dials `--server-id` through the configured relays (every custom relay as a hint), proves `--private-key-file`, checks the echo. Exits `0` on `Echo OK`, `3` when the server rejects the key, `1` otherwise. Logs the path it connected over (`via Relay ` / `via Direct `). | +| `keygen` | writes a client key in the shared flexaccess-keys format (mode 0600) and its `authorized_keys` entry | +| `fake-relay` | a relay that answers the net-report probe (`GET /ping`) but refuses relay connections — the outage shape the failover exists for | +| `delay-proxy` | a TCP proxy adding latency to each new connection, so the relay behind it always measures slower | +| `pick-port` | free localhost ports | + +Everything a product adds — its ALPN, QUIC tuning, config files, forwarding — +is left out, so a failure here is a failure of this crate or of iroh. + +## Suites + +| Script | What it checks | Needs | +|---|---|---| +| `run_e2e.sh` | **Auth + connectivity.** An unlisted key is rejected (exit 3, explicit rejection, server logs why); an authorized client authenticates and gets its echo; a second client on the *same* key uses a *distinct* ephemeral iroh identity; under `--relay-only`, every connection ran through a relay. | internet for the default relays; or `--local-relays` / `--relay-url` ×2 for a fully offline run | +| `run_relay_failover.sh` | **Relay failover**, fully offline against two local `iroh-relay --dev` instances, relay-only. Phase A: relays down *before* startup (per-relay probe: startup fails only when none is reachable; a single custom relay is rejected). Phase B: a relay dies *after* startup (iroh re-homes on its own; both down fails new clients; both back recovers). Phase C: the home relay answers probes but refuses connections (the in-place failover removes it after 60 s, the server homes on the other relay without restarting, and the restore probe puts it back once it is connectable again). | `iroh-relay` | + +```sh +cargo install iroh-relay --features server # one-time + +./e2e/run_e2e.sh # default relays + n0 discovery (internet) +./e2e/run_e2e.sh --local-relays # two local relays, offline, direct paths allowed +./e2e/run_e2e.sh --local-relays --relay-only # ... relay paths only +./e2e/run_e2e.sh --relay-url URL --relay-url URL [--relay-only] # your own relays + +./e2e/run_relay_failover.sh # ~6 minutes: 60 s failover window + 90 s restore probe +``` + +Each suite prints a `PASS`/`FAIL` line per scenario and exits non-zero on +any failure. The scenario list and contract are at the top of each script. + +### Environment + +| Variable | Default | Meaning | +|---|---|---| +| `E2E_BIN` | `target/debug/examples/e2e` (rebuilt by cargo) | The harness binary | +| `IROH_RELAY_BIN` | `iroh-relay` on `PATH` | The relay binary | +| `READY_TIMEOUT` | `60` | Seconds to wait for a process to become ready | +| `KEEP_LOGS` | `0` | `1` keeps the working directory (per-process logs; it holds the generated test keys, nothing else) | +| `RELAY_URL` | unset | `run_e2e.sh` only: whitespace-separated fallback for `--relay-url` | + +`RUST_LOG` is honored by the harness (default `info`; the crate's own log +lines are what the scripts assert on). + +## CI + +The `e2e` job in `.github/workflows/ci.yml` runs the offline suites: +`run_e2e.sh --local-relays`, `run_e2e.sh --local-relays --relay-only`, and +`run_relay_failover.sh`. The default-relay run needs n0's public relays and +DNS and is left to a developer machine. diff --git a/e2e/lib.sh b/e2e/lib.sh new file mode 100644 index 0000000..2732663 --- /dev/null +++ b/e2e/lib.sh @@ -0,0 +1,205 @@ +# Sourced by the e2e scripts: the harness binary, process management, log +# waits, ports, keys, and local iroh-relay instances. +# +# Requirements: bash, cargo (to build the harness), and iroh-relay for the +# suites that run relays locally (`cargo install iroh-relay --features server`). +# +# Environment overrides honored by every suite: +# E2E_BIN path to a built harness (default: cargo builds examples/e2e) +# IROH_RELAY_BIN path to the iroh-relay binary (default: iroh-relay on PATH) +# KEEP_LOGS set to 1 to keep the working directory after the run +# READY_TIMEOUT seconds to wait for each process to become ready (default 60) + +E2E_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_DIR="$(cd "$E2E_DIR/.." && pwd)" +READY_TIMEOUT="${READY_TIMEOUT:-60}" + +log() { printf '==> %s\n' "$*"; } +note() { printf ' %s\n' "$*"; } +die() { echo "ERROR: $*" >&2; exit 1; } + +# --------------------------------------------------------------------------- +# The harness binary +# --------------------------------------------------------------------------- +if [[ -n "${E2E_BIN:-}" ]]; then + BIN="$E2E_BIN" +else + # Always rebuild: cargo is a no-op when the harness is current, and a stale + # binary would test the wrong crate. + log "Building the e2e harness (cargo build --example e2e)..." + (cd "$REPO_DIR" && cargo build -q --example e2e) || die "building the e2e harness failed" + BIN="$REPO_DIR/target/debug/examples/e2e" +fi +[[ -x "$BIN" ]] || die "e2e harness not found at $BIN" + +require_iroh_relay() { + RELAY_BIN="${IROH_RELAY_BIN:-$(command -v iroh-relay || true)}" + [[ -n "$RELAY_BIN" && -x "$RELAY_BIN" ]] || + die "iroh-relay not found. Install with: cargo install iroh-relay --features server" +} + +# --------------------------------------------------------------------------- +# Working directory + process management +# --------------------------------------------------------------------------- +WORK="$(mktemp -d)" +declare -a PIDS=() + +cleanup() { + local status=$? + for pid in "${PIDS[@]:-}"; do + [[ -n "$pid" ]] || continue + kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true + done + wait 2>/dev/null || true + if [[ "${KEEP_LOGS:-0}" == "1" ]]; then + echo "==> Logs kept in $WORK" + else + rm -rf "$WORK" + fi + exit "$status" +} +trap cleanup EXIT INT TERM + +# Start a background process, in its own session where setsid exists so the +# whole process group can be killed. Args: . Records +# the PID in PIDS and BG_PID. +start_bg() { + local logfile="$1"; shift + if command -v setsid >/dev/null 2>&1; then + setsid "$@" >"$logfile" 2>&1 & + else + "$@" >"$logfile" 2>&1 & + fi + BG_PID=$! + PIDS+=("$BG_PID") +} + +kill_pid() { + local pid="$1" + [[ -n "$pid" ]] || return 0 + kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true + # Reap so a port is really free before a restart. + for _ in $(seq 1 50); do + kill -0 "$pid" 2>/dev/null || return 0 + sleep 0.1 + done + kill -KILL -- "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true +} + +# Wait until $1 (a log file) contains the regex $2, or time out after $3 secs. +wait_for_log() { + local logfile="$1" pattern="$2" timeout="$3" + local max_attempts=$(( timeout * 2 )) attempt=0 + while (( attempt < max_attempts )); do + if [[ -f "$logfile" ]] && grep -Eq "$pattern" "$logfile"; then + return 0 + fi + sleep 0.5 + attempt=$(( attempt + 1 )) + done + return 1 +} + +# Like wait_for_log, but gives up early (rc 2) if process $1 exits first. +wait_for_log_or_death() { + local pid="$1" logfile="$2" pattern="$3" timeout="$4" + local max_attempts=$(( timeout * 2 )) attempt=0 + while (( attempt < max_attempts )); do + if [[ -f "$logfile" ]] && grep -Eq "$pattern" "$logfile"; then + return 0 + fi + if ! kill -0 "$pid" 2>/dev/null; then + # One last look: the pattern may have landed just before exit. + grep -Eq "$pattern" "$logfile" 2>/dev/null && return 0 + return 2 + fi + sleep 0.5 + attempt=$(( attempt + 1 )) + done + return 1 +} + +# Wait until something accepts TCP connections on 127.0.0.1:$1, or time out +# after $2 seconds. +wait_for_tcp_port() { + local port="$1" timeout="$2" + local deadline=$(( SECONDS + timeout )) + while (( SECONDS < deadline )); do + if (exec 3<>"/dev/tcp/127.0.0.1/$port") 2>/dev/null; then + return 0 + fi + sleep 0.25 + done + return 1 +} + +dump_log() { + echo "----- $1 -----" >&2 + cat "$1" >&2 || true +} + +# --------------------------------------------------------------------------- +# Ports, keys, identities +# --------------------------------------------------------------------------- + +# Print $1 distinct free localhost ports, one per line. +pick_ports() { + "$BIN" pick-port --count "$1" +} + +# A fresh server identity: any 32 bytes are an Ed25519 seed. The harness +# logs the matching EndpointId at startup (see server_endpoint_id). +new_server_secret() { + head -c 32 /dev/urandom | base64 | tr -d '\n' +} + +# The EndpointId a server logged at startup. +server_endpoint_id() { + sed -nE 's/.*EndpointId: ([0-9a-f]+).*/\1/p' "$1" | head -1 +} + +# Write a client key named $1 to $2 and append its entry to $3 (optional). +keygen() { + local comment="$1" private_key_file="$2" authorized_keys="${3:-}" + if [[ -n "$authorized_keys" ]]; then + "$BIN" keygen "$comment" --private-key-file "$private_key_file" \ + --authorized-keys "$authorized_keys" >/dev/null + else + "$BIN" keygen "$comment" --private-key-file "$private_key_file" >/dev/null + fi +} + +# --------------------------------------------------------------------------- +# Local iroh-relay instances (dev mode, plain HTTP; configs hold no secrets) +# --------------------------------------------------------------------------- +declare -A RELAY_PORT=() +declare -A RELAY_PID=() + +# Register relay $1 on port $2 and write its config. +define_relay() { + local num="$1" port="$2" + RELAY_PORT[$num]="$port" + cat > "$WORK/relay$num.toml" <&2; exit 2; } + RELAY_URLS+=("$1") + ;; + --relay-url=*) RELAY_URLS+=("${1#*=}") ;; + --local-relays) LOCAL_RELAYS=1 ;; + --relay-only) RELAY_ONLY=1 ;; + -h|--help) usage; exit 0 ;; + *) echo "ERROR: unknown argument: $1" >&2; usage >&2; exit 2 ;; + esac + shift +done + +if [[ ${#RELAY_URLS[@]} -eq 0 && -n "${RELAY_URL:-}" ]]; then + read -r -a RELAY_URLS <<<"$RELAY_URL" +fi +if [[ "$LOCAL_RELAYS" == 1 && ${#RELAY_URLS[@]} -gt 0 ]]; then + echo "ERROR: --local-relays and --relay-url are exclusive" >&2 + exit 2 +fi +if [[ ${#RELAY_URLS[@]} -eq 1 ]]; then + echo "ERROR: custom relays need at least two --relay-url values" >&2 + exit 2 +fi +if [[ "$RELAY_ONLY" == 1 && "$LOCAL_RELAYS" == 0 && ${#RELAY_URLS[@]} -eq 0 ]]; then + echo "ERROR: --relay-only requires custom relays (--relay-url or --local-relays)" >&2 + exit 2 +fi + +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" + +# --------------------------------------------------------------------------- +# Relays +# --------------------------------------------------------------------------- +if [[ "$LOCAL_RELAYS" == 1 ]]; then + require_iroh_relay + mapfile -t PORTS < <(pick_ports 2) + RELAY1_PORT="${PORTS[0]}" + RELAY2_PORT="${PORTS[1]}" + define_relay 1 "$RELAY1_PORT" + define_relay 2 "$RELAY2_PORT" + log "Starting two local relays..." + start_relay 1 + start_relay 2 + RELAY_URLS=("http://127.0.0.1:$RELAY1_PORT" "http://127.0.0.1:$RELAY2_PORT") +fi + +declare -a RELAY_ARGS=() +for url in "${RELAY_URLS[@]:-}"; do + [[ -n "$url" ]] && RELAY_ARGS+=(--relay-url "$url") +done +if [[ "$RELAY_ONLY" == 1 ]]; then + RELAY_ARGS+=(--relay-only) +fi +if [[ ${#RELAY_URLS[@]} -gt 0 ]]; then + log "Using custom relays (internet discovery off): ${RELAY_URLS[*]}" +else + log "Using the default relays and n0 discovery (needs internet)" +fi +[[ "$RELAY_ONLY" == 1 ]] && log "Relay-only: no direct paths" + +# --------------------------------------------------------------------------- +# Keys and identity +# --------------------------------------------------------------------------- +keygen "e2e client" "$WORK/client.key" "$WORK/authorized_keys" +keygen "unlisted e2e client" "$WORK/unlisted.key" +SECRET="$(new_server_secret)" + +# --------------------------------------------------------------------------- +# Server +# --------------------------------------------------------------------------- +SERVER_LOG="$WORK/server.log" +log "Starting the server..." +E2E_SERVER_SECRET="$SECRET" start_bg "$SERVER_LOG" \ + "$BIN" server --authorized-keys "$WORK/authorized_keys" ${RELAY_ARGS[@]+"${RELAY_ARGS[@]}"} +SERVER_PID="$BG_PID" +unset SECRET +wait_for_log_or_death "$SERVER_PID" "$SERVER_LOG" "Waiting for clients to connect" "$READY_TIMEOUT" || { + dump_log "$SERVER_LOG" + die "the server did not come up" +} +SERVER_ID="$(server_endpoint_id "$SERVER_LOG")" +[[ -n "$SERVER_ID" ]] || die "the server did not log its EndpointId" +log "Server EndpointId: $SERVER_ID" + +# Run one client to completion. Args: . +# Returns the client's exit status. +run_client() { + local logfile="$1" key="$2" message="$3" rc=0 + "$BIN" client --server-id "$SERVER_ID" --private-key-file "$key" \ + --message "$message" ${RELAY_ARGS[@]+"${RELAY_ARGS[@]}"} >"$logfile" 2>&1 || rc=$? + return "$rc" +} + +RESULT=0 +declare -a SUMMARY=() +record() { + local id="$1" ok="$2" + if [[ "$ok" -eq 0 ]]; then + SUMMARY+=("PASS $id"); log "[$id] PASS" + else + SUMMARY+=("FAIL $id"); log "[$id] FAIL"; RESULT=1 + fi +} + +# --------------------------------------------------------------------------- +# 1. An unlisted key is rejected +# --------------------------------------------------------------------------- +log "[unlisted-key] a key missing from authorized_keys is rejected" +rc=0 +run_client "$WORK/client_unlisted.log" "$WORK/unlisted.key" "should-not-echo" || rc=$? +ok=0 +if [[ "$rc" -ne 3 ]]; then + note "client exited with $rc instead of auth status 3"; ok=1 +fi +if ! grep -q "Authentication rejected: Invalid authentication proof" "$WORK/client_unlisted.log"; then + note "client did not receive the explicit rejection"; ok=1 +fi +if grep -q "Authenticated as" "$WORK/client_unlisted.log"; then + note "client with an unlisted key authenticated"; ok=1 +fi +if ! grep -Eq "Rejected client [0-9a-f]+: key ed25519-pub:[A-Za-z0-9_-]+ is not authorized" "$SERVER_LOG"; then + note "server did not log the unauthorized key"; ok=1 +fi +[[ "$ok" -eq 0 ]] || dump_log "$WORK/client_unlisted.log" +record unlisted-key "$ok" + +# --------------------------------------------------------------------------- +# 2./3. Two clients on the same key, distinct ephemeral identities +# --------------------------------------------------------------------------- +log "[echo] an authorized client authenticates and gets its message echoed" +ok=0 +run_client "$WORK/client_1.log" "$WORK/client.key" "hello-1-$(date +%s%N)" || { + note "first client failed (rc=$?)"; ok=1 +} +grep -q "Echo OK" "$WORK/client_1.log" || { note "first client reported no echo"; ok=1; } +[[ "$ok" -eq 0 ]] || dump_log "$WORK/client_1.log" +record echo "$ok" + +log "[shared-key] a second client with the same key uses a distinct ephemeral identity" +ok=0 +run_client "$WORK/client_2.log" "$WORK/client.key" "hello-2-$(date +%s%N)" || { + note "second client failed (rc=$?)"; ok=1 +} +grep -q "Echo OK" "$WORK/client_2.log" || { note "second client reported no echo"; ok=1; } +mapfile -t AUTHENTICATED_IDS < <( + sed -nE 's/.*Client ([0-9a-f]+) authenticated successfully as e2e client.*/\1/p' \ + "$SERVER_LOG" | sort -u +) +if (( ${#AUTHENTICATED_IDS[@]} < 2 )); then + note "server saw ${#AUTHENTICATED_IDS[@]} distinct client identities, expected 2"; ok=1 +fi +[[ "$ok" -eq 0 ]] || { dump_log "$WORK/client_2.log"; dump_log "$SERVER_LOG"; } +record shared-key "$ok" + +# --------------------------------------------------------------------------- +# 4. Relay-only really went through a relay +# --------------------------------------------------------------------------- +if [[ "$RELAY_ONLY" == 1 ]]; then + log "[relay-path] relay-only connections use a relay path" + ok=0 + for logfile in "$WORK/client_1.log" "$WORK/client_2.log"; do + if ! grep -Eq "Connected to [0-9a-f]+ via Relay " "$logfile"; then + note "$(basename "$logfile") did not connect via a relay"; ok=1 + fi + if grep -Eq "via .*Direct " "$logfile"; then + note "$(basename "$logfile") used a direct path under --relay-only"; ok=1 + fi + done + record relay-path "$ok" +fi + +# The server must still be up after all of that. +if ! kill -0 "$SERVER_PID" 2>/dev/null; then + log "[server-alive] FAIL: the server exited" + dump_log "$SERVER_LOG" + RESULT=1 +fi + +echo +log "Auth + connectivity e2e summary:" +for line in "${SUMMARY[@]}"; do note "$line"; done +if [[ "$RESULT" -eq 0 ]]; then + log "E2E RESULT: ALL PASS ✅" +else + log "E2E RESULT: FAILURES ❌ (re-run with KEEP_LOGS=1 to inspect $WORK)" +fi +exit "$RESULT" diff --git a/e2e/run_relay_failover.sh b/e2e/run_relay_failover.sh new file mode 100755 index 0000000..89e1c2e --- /dev/null +++ b/e2e/run_relay_failover.sh @@ -0,0 +1,501 @@ +#!/usr/bin/env bash +# +# Relay failover end-to-end test for flexaccess-iroh (relay-only, no internet). +# +# Runs TWO local iroh-relay instances (`--dev` mode, plain HTTP) and exercises +# relay failures against the e2e harness (examples/e2e). Servers and clients +# are each given an explicit relay list per scenario. Custom relays disable +# internet discovery, so nothing here touches public iroh infrastructure. +# +# Contract under test: a custom relay set holds at least TWO distinct relays, +# because a server rides out a relay outage by moving onto another configured +# relay in place (no rebuild, no dropped identity or connections). Every +# configured relay is probed individually at startup; a relay that is down is +# a warning, and startup fails only when none is reachable. Clients dial with +# every configured relay as a hint, so a server homed on any of them is reachable. +# +# Phase A - relay offline BEFORE startup (the per-relay startup probe): +# A0 both relays down; server configured with both ..... startup fails (negative) +# A1 only relay2 up; server and client with both ....... both start (warning +# for relay1); the client connects via relay2; echo passes +# A2 a single custom relay is rejected as configuration: server and client +# configured with ONLY relay2 ........................ startup fails (negative) +# +# Phase B - a relay dies AFTER startup (iroh's own re-homing): +# B1 both relays up; server and client with both relays; connects; echo passes +# B2 the server's home relay is killed; the server stays up and re-homes onto +# the survivor on its own (net_report re-probes every ~20-26s); a new +# client configured with both relays connects; echo passes +# B3 the surviving relay is killed too (both down); a new client fails (negative) +# B4 both relays are restarted; a new client with both relays connects again +# (server relay reconnect + re-home); echo passes +# +# Phase C - the home relay stays "healthy" for net_report but cannot be connected +# (the in-place home-relay failover, flexaccess_iroh::relay_failover): +# C0 relay1 direct, relay2 behind a proxy that adds latency; the server homes +# on relay1 deterministically; a client connects; echo passes +# C1 relay1 is replaced, on the same port, by a fake that answers the net-report +# probe (/ping) but refuses relay connections. iroh keeps preferring it, so +# nothing re-homes on its own; after 60s the failover removes it from the +# relay map, the forced net report homes the server on relay2, and a new +# client connects through relay2; echo passes. The server process and its +# endpoint never restart. +# C2 the real relay1 comes back on its port; the failover's restore probe puts +# it back in the relay map (checked every 90s), the server moves back onto +# it, and a client connects via relay1; echo passes +# +# Requirements: cargo, iroh-relay (cargo install iroh-relay --features server). +# +# Usage: +# ./e2e/run_relay_failover.sh +# +# Environment: E2E_BIN, IROH_RELAY_BIN, KEEP_LOGS, READY_TIMEOUT (see lib.sh). +# +set -euo pipefail + +# shellcheck source=lib.sh +source "$(dirname "${BASH_SOURCE[0]}")/lib.sh" +require_iroh_relay + +# --------------------------------------------------------------------------- +# Ports, identity, client authentication key +# --------------------------------------------------------------------------- +mapfile -t PORTS < <(pick_ports 4) +RELAY1_PORT="${PORTS[0]}" +RELAY2_PORT="${PORTS[1]}" +PROXY_PORT="${PORTS[2]}" +DEAD_PORT="${PORTS[3]}" +RELAY1_URL="http://127.0.0.1:$RELAY1_PORT" +RELAY2_URL="http://127.0.0.1:$RELAY2_PORT" +# relay2 as seen through the latency-adding proxy (phase C). +PROXY_URL="http://127.0.0.1:$PROXY_PORT" +# A relay URL nothing ever listens on: fills the second slot of a client that +# must only use one live relay, since a single custom relay is rejected. +DEAD_URL="http://127.0.0.1:$DEAD_PORT" +log "Relays: relay1=$RELAY1_URL relay2=$RELAY2_URL proxy->relay2=$PROXY_URL" + +define_relay 1 "$RELAY1_PORT" +define_relay 2 "$RELAY2_PORT" + +SECRET="$(new_server_secret)" +keygen "failover e2e client" "$WORK/client.key" "$WORK/authorized_keys" + +# --------------------------------------------------------------------------- +# Fake relay and delay proxy (phase C) +# --------------------------------------------------------------------------- + +# A relay that answers the net-report probe but refuses relay connections, on +# relay1's port. +FAKE_RELAY_PID="" +start_fake_relay() { + local logfile="$WORK/fake_relay.$(date +%s%N).log" + start_bg "$logfile" "$BIN" fake-relay --port "$RELAY1_PORT" + FAKE_RELAY_PID="$BG_PID" + wait_for_log "$logfile" "READY fake relay" 30 || { + dump_log "$logfile" + die "the fake relay did not start" + } + note "fake relay up on relay1's port $RELAY1_PORT (pid $FAKE_RELAY_PID)" +} + +stop_fake_relay() { + kill_pid "$FAKE_RELAY_PID" + FAKE_RELAY_PID="" + note "fake relay stopped" +} + +# relay2 behind a proxy that delays each new connection, so net_report always +# measures it as the slower relay. +PROXY_PID="" +PROXY_DELAY_MS=40 +start_delay_proxy() { + local logfile="$WORK/delay_proxy.$(date +%s%N).log" + start_bg "$logfile" "$BIN" delay-proxy --listen "$PROXY_PORT" \ + --upstream "$RELAY2_PORT" --delay-ms "$PROXY_DELAY_MS" + PROXY_PID="$BG_PID" + wait_for_log "$logfile" "READY delay proxy" 30 || { + dump_log "$logfile" + die "the delay proxy did not start" + } + note "delay proxy up: $PROXY_URL -> relay2 (+${PROXY_DELAY_MS}ms per connection)" +} + +stop_delay_proxy() { + kill_pid "$PROXY_PID" + PROXY_PID="" + note "delay proxy stopped" +} + +# --------------------------------------------------------------------------- +# Server management +# --------------------------------------------------------------------------- +SERVER_PID="" +SERVER_LOG="" +SERVER_ID="" + +# Turn relay URLs into harness arguments. Args: ... +relay_args() { + local url + for url in "$@"; do + printf -- '--relay-url\0%s\0' "$url" + done +} + +# Start the server in relay-only mode. Args: ... +start_server() { + SERVER_LOG="$WORK/server.$(date +%s%N).log" + local -a args=() + mapfile -d '' -t args < <(relay_args "$@") + E2E_SERVER_SECRET="$SECRET" start_bg "$SERVER_LOG" \ + "$BIN" server --relay-only --authorized-keys "$WORK/authorized_keys" "${args[@]}" + SERVER_PID="$BG_PID" +} + +# Start a server that is EXPECTED to fail at startup for the reason matching +# regex $1. Passes when the process reports that failure and never becomes +# ready. Args: ... +expect_server_start_failure() { + local pattern="$1"; shift + local rc=0 + start_server "$@" + wait_for_log_or_death "$SERVER_PID" "$SERVER_LOG" "$pattern" "$READY_TIMEOUT" || rc=$? + # rc 0 (error logged) or 2 (process died after logging) both mean it gave + # up; verify it never became ready. + if grep -Eq "Waiting for clients to connect" "$SERVER_LOG"; then + rc=1 + elif [[ "$rc" -eq 2 ]]; then + rc=0 + fi + [[ "$rc" -eq 0 ]] || dump_log "$SERVER_LOG" + stop_server + return "$rc" +} + +# Start a server that is EXPECTED to come up. Args: ... +expect_server_ready() { + local rc=0 + start_server "$@" + wait_for_log_or_death "$SERVER_PID" "$SERVER_LOG" \ + "Waiting for clients to connect" "$READY_TIMEOUT" || rc=1 + if [[ "$rc" -eq 0 ]]; then + SERVER_ID="$(server_endpoint_id "$SERVER_LOG")" + [[ -n "$SERVER_ID" ]] || { note "server did not log its EndpointId"; rc=1; } + fi + [[ "$rc" -eq 0 ]] || dump_log "$SERVER_LOG" + return "$rc" +} + +stop_server() { + kill_pid "$SERVER_PID" + SERVER_PID="" +} + +# The port of the relay the server last reported as its connected home relay. +# The failover logs "Home relay: connected" on a change while healthy +# and "Home relay connection restored on after Ns" when an outage ends; +# either names the current home relay. +server_home_relay_port() { + grep -Eo "Home relay: http://127\.0\.0\.1:[0-9]+/ connected|Home relay connection restored on http://127\.0\.0\.1:[0-9]+/" "$SERVER_LOG" | + tail -1 | sed -E 's|.*:([0-9]+)/.*|\1|' +} + +# Wait until the server reports its home relay connected on port $1. +wait_for_home_relay() { + local port="$1" timeout="$2" + local max_attempts=$(( timeout * 2 )) attempt=0 + while (( attempt < max_attempts )); do + [[ "$(server_home_relay_port)" == "$port" ]] && return 0 + sleep 0.5 + attempt=$(( attempt + 1 )) + done + return 1 +} + +# --------------------------------------------------------------------------- +# Client runs +# --------------------------------------------------------------------------- +CLIENT_LOG="" + +# Run one relay-only client to completion. Args: ... +# Returns the client's exit status. +run_client() { + local logfile="$1"; shift + local -a args=() + mapfile -d '' -t args < <(relay_args "$@") + local rc=0 + "$BIN" client --relay-only --server-id "$SERVER_ID" \ + --private-key-file "$WORK/client.key" \ + --message "failover-$(date +%s%N)" "${args[@]}" >"$logfile" 2>&1 || rc=$? + return "$rc" +} + +# Connect and push one echo through, retrying with a fresh client process: +# after a relay failure the server needs a re-probe cycle (~20-26s) to re-home. +# Args: ... +connect_and_echo() { + local attempts="$1"; shift + local attempt rc + for (( attempt = 1; attempt <= attempts; attempt++ )); do + CLIENT_LOG="$WORK/client.$(date +%s%N).log" + rc=0 + run_client "$CLIENT_LOG" "$@" || rc=$? + if [[ "$rc" -eq 0 ]] && grep -q "Echo OK" "$CLIENT_LOG"; then + note "client connected $(grep -Eo 'via .*' "$CLIENT_LOG" | head -1)" + return 0 + fi + note "attempt $attempt/$attempts failed (rc=$rc), retrying..." + sleep 3 + done + dump_log "$CLIENT_LOG" + return 1 +} + +# The message the startup relay probe emits when EVERY configured relay is +# down (one dead relay is only a warning). Negative scenarios must fail in +# the expected way; requiring the message keeps an unrelated startup failure +# (bad key, malformed config) from passing as the expected one. +RELAY_PROBE_FAILURE='all [0-9]+ custom relays failed to come online' +# The config error for fewer than two distinct custom relays. +SINGLE_RELAY_REJECTED='at least 2 distinct relay_urls' + +# Run a client that is EXPECTED to fail for the reason matching regex $1. +# Passes when it exits non-zero without ever authenticating AND reports that +# failure. Args: ... +expect_connect_failure() { + local pattern="$1"; shift + local logfile="$WORK/client.$(date +%s%N).log" + local rc=0 + run_client "$logfile" "$@" || rc=$? + if [[ "$rc" -eq 0 ]] || grep -q "Authenticated as" "$logfile"; then + note "client unexpectedly connected (rc=$rc)" + dump_log "$logfile" + return 1 + fi + if ! grep -Eq "$pattern" "$logfile"; then + note "client failed without the expected reason ($pattern)" + dump_log "$logfile" + return 1 + fi + return 0 +} + +# --------------------------------------------------------------------------- +# Scenario bookkeeping +# --------------------------------------------------------------------------- +RESULT=0 +declare -a SUMMARY=() + +scenario() { + log "[$1] $2" +} + +record() { + local id="$1" ok="$2" + if [[ "$ok" -eq 0 ]]; then + SUMMARY+=("PASS $id"); log "[$id] PASS" + else + SUMMARY+=("FAIL $id"); log "[$id] FAIL"; RESULT=1 + fi +} + +# =========================================================================== +# Phase A - relays offline BEFORE startup (the per-relay startup probe) +# =========================================================================== + +scenario A0 "both relays down: server configured with both fails to start" +rc=0 +expect_server_start_failure "$RELAY_PROBE_FAILURE" "$RELAY1_URL" "$RELAY2_URL" || rc=1 +record A0 "$rc" + +scenario A1 "relay1 down: server and client configured with both start on relay2 alone" +start_relay 2 +rc=0 +expect_server_ready "$RELAY1_URL" "$RELAY2_URL" || rc=1 +if [[ "$rc" -eq 0 ]] && ! grep -Eq "1 of 2 custom relays failed to come online" "$SERVER_LOG"; then + note "server did not warn about the dead relay" + rc=1 +fi +if [[ "$rc" -eq 0 ]]; then + connect_and_echo 3 "$RELAY1_URL" "$RELAY2_URL" || rc=1 +fi +if [[ "$rc" -eq 0 ]] && ! grep -Eq "1 of 2 custom relays failed to come online" "$CLIENT_LOG"; then + note "client did not warn about the dead relay" + rc=1 +fi +if [[ "$rc" -eq 0 ]] && ! grep -Eq "Connected to [0-9a-f]+ via Relay $RELAY2_URL/" "$CLIENT_LOG"; then + note "client did not connect through relay2" + rc=1 +fi +record A1 "$rc" +stop_server + +scenario A2 "a single custom relay is rejected: server and client with only relay2 fail to start" +rc=0 +expect_server_start_failure "$SINGLE_RELAY_REJECTED" "$RELAY2_URL" || rc=1 +expect_connect_failure "$SINGLE_RELAY_REJECTED" "$RELAY2_URL" || rc=1 +record A2 "$rc" + +stop_relay 2 + +# =========================================================================== +# Phase B - a relay dies AFTER client/server are connected (iroh re-homes) +# =========================================================================== + +scenario B1 "both relays up: client with both relays connects" +start_relay 1 +start_relay 2 +rc=0 +expect_server_ready "$RELAY1_URL" "$RELAY2_URL" || rc=1 +HOME_RELAY_NUM="" +if [[ "$rc" -eq 0 ]]; then + connect_and_echo 3 "$RELAY1_URL" "$RELAY2_URL" || rc=1 +fi +if [[ "$rc" -eq 0 ]]; then + case "$(server_home_relay_port)" in + "$RELAY1_PORT") HOME_RELAY_NUM=1 ;; + "$RELAY2_PORT") HOME_RELAY_NUM=2 ;; + *) note "server did not report a connected home relay"; rc=1 ;; + esac + note "server home relay is relay$HOME_RELAY_NUM" +fi +record B1 "$rc" + +scenario B2 "kill the server's home relay: server re-homes on its own, a new client connects via the survivor" +rc=0 +if [[ -n "$HOME_RELAY_NUM" ]]; then + SURVIVOR_NUM=$(( 3 - HOME_RELAY_NUM )) + SURVIVOR_PORT="${RELAY_PORT[$SURVIVOR_NUM]}" + stop_relay "$HOME_RELAY_NUM" + # Losing a relay at runtime must NOT take the already-started server down. + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + note "server exited when a relay died at runtime" + rc=1 + fi + # The dead relay stops answering net_report's probe, so iroh itself moves + # the home relay within a re-probe cycle, no failover action needed. + if [[ "$rc" -eq 0 ]]; then + wait_for_home_relay "$SURVIVOR_PORT" 60 || { note "server did not re-home onto relay$SURVIVOR_NUM"; rc=1; } + fi + if [[ "$rc" -eq 0 ]] && grep -Eq "Removed .* from the relay map" "$SERVER_LOG"; then + note "the failover acted although iroh re-homed on its own" + rc=1 + fi + # The new client lists both relays (one is dead: a warning at its probe). + if [[ "$rc" -eq 0 ]]; then + connect_and_echo 4 "$RELAY1_URL" "$RELAY2_URL" || rc=1 + fi +else + rc=1 +fi +record B2 "$rc" + +scenario B3 "kill the surviving relay too (both down): new client fails" +rc=0 +if [[ -n "${SURVIVOR_NUM:-}" ]]; then + stop_relay "$SURVIVOR_NUM" + expect_connect_failure "$RELAY_PROBE_FAILURE" "$RELAY1_URL" "$RELAY2_URL" || rc=1 +else + rc=1 +fi +record B3 "$rc" + +scenario B4 "restart both relays: server recovers and a new client connects" +rc=0 +start_relay 1 +start_relay 2 +# The server's relay actors reconnect with backoff; allow several attempts. +connect_and_echo 8 "$RELAY1_URL" "$RELAY2_URL" || rc=1 +record B4 "$rc" + +stop_server +stop_relay 1 +stop_relay 2 + +# =========================================================================== +# Phase C - the home relay answers probes but cannot be connected (in-place failover) +# =========================================================================== + +scenario C0 "relay1 direct, relay2 behind a slow proxy: server homes on relay1" +rc=0 +start_relay 1 +start_relay 2 +start_delay_proxy +expect_server_ready "$RELAY1_URL" "$PROXY_URL" || rc=1 +if [[ "$rc" -eq 0 ]]; then + wait_for_home_relay "$RELAY1_PORT" 30 || { note "server did not home on relay1"; rc=1; } +fi +if [[ "$rc" -eq 0 ]]; then + connect_and_echo 3 "$RELAY1_URL" "$PROXY_URL" || rc=1 +fi +record C0 "$rc" + +scenario C1 "relay1 becomes a fake that answers probes but refuses connections: the failover moves the server onto relay2 in place" +rc=0 +stop_relay 1 +start_fake_relay +wait_for_log "$SERVER_LOG" "No connected home relay" 60 || { note "server never noticed the relay loss"; rc=1; } +if [[ "$rc" -eq 0 ]]; then + # 60s outage window, then the wedged relay is taken out of the relay map. + wait_for_log "$SERVER_LOG" "Removed $RELAY1_URL/ from the relay map" 90 || { + note "the failover did not remove relay1 from the relay map"; rc=1; } +fi +if [[ "$rc" -eq 0 ]]; then + wait_for_log "$SERVER_LOG" "Home relay connection restored on $PROXY_URL/" 60 || { + note "server did not home on relay2 after the failover"; rc=1; } +fi +if [[ "$rc" -eq 0 ]]; then + # iroh must not have re-homed on its own before the failover acted. + removed_line="$(grep -En "Removed $RELAY1_URL/ from the relay map" "$SERVER_LOG" | head -1 | cut -d: -f1)" + restored_line="$(grep -En "Home relay connection restored on $PROXY_URL/" "$SERVER_LOG" | head -1 | cut -d: -f1)" + if (( restored_line < removed_line )); then + note "server re-homed before the failover acted (lines $restored_line < $removed_line)" + rc=1 + fi +fi +if [[ "$rc" -eq 0 ]] && ! kill -0 "$SERVER_PID" 2>/dev/null; then + note "server process died" + rc=1 +fi +# A new client reaches the server through relay2. Its second relay slot is a +# dead port rather than the fake: a client that homed on the fake could never +# come online, and the failover runs only on the server. +if [[ "$rc" -eq 0 ]]; then + connect_and_echo 3 "$PROXY_URL" "$DEAD_URL" || rc=1 +fi +[[ "$rc" -eq 0 ]] || dump_log "$SERVER_LOG" +record C1 "$rc" + +scenario C2 "the real relay1 returns: the restore probe puts it back and the server moves back onto it" +rc=0 +stop_fake_relay +start_relay 1 +# The restore probe runs every 90s after the removal; allow for one full +# interval plus the 10s probe, then a net-report cycle for the move back. +wait_for_log "$SERVER_LOG" "$RELAY1_URL/ is connectable again and back in the relay map" 150 || { + note "relay1 was not restored to the relay map"; rc=1; } +if [[ "$rc" -eq 0 ]]; then + wait_for_home_relay "$RELAY1_PORT" 60 || { note "server did not move back onto relay1"; rc=1; } +fi +if [[ "$rc" -eq 0 ]]; then + connect_and_echo 3 "$RELAY1_URL" "$PROXY_URL" || rc=1 +fi +[[ "$rc" -eq 0 ]] || dump_log "$SERVER_LOG" +record C2 "$rc" + +stop_server +stop_delay_proxy +stop_relay 1 +stop_relay 2 + +# --------------------------------------------------------------------------- +# Summary +# --------------------------------------------------------------------------- +echo +log "Relay failover e2e summary:" +for line in "${SUMMARY[@]}"; do note "$line"; done +if [[ "$RESULT" -eq 0 ]]; then + log "E2E RESULT: ALL PASS ✅" +else + log "E2E RESULT: FAILURES ❌ (re-run with KEEP_LOGS=1 to inspect $WORK)" +fi +exit "$RESULT" diff --git a/examples/e2e/main.rs b/examples/e2e/main.rs new file mode 100644 index 0000000..95c6b89 --- /dev/null +++ b/examples/e2e/main.rs @@ -0,0 +1,523 @@ +//! The end-to-end harness for this crate: the smallest program that puts +//! every module through real relays, driven by the scripts in `e2e/`. +//! +//! It is deliberately not an application. A `server` binds an endpoint with +//! the shared builder, runs the shared home-relay failover beside its accept +//! loop, and answers one request per connection: the endpoint-bound auth +//! transcript from [`flexaccess_iroh::auth`] followed by an echo of the +//! client's message. A `client` builds an ephemeral endpoint the same way, +//! dials the server through the configured relays, proves its key, and exits +//! `0` on a clean echo or [`EXIT_AUTH_REJECTED`] when the server refuses the +//! key. Everything an application would add on top — a product ALPN, QUIC +//! tuning, config files, forwarding — is left out so a failure here is a +//! failure of this crate or of iroh, never of a product. +//! +//! The remaining subcommands are the test fixtures the relay-failover suite +//! needs, kept in Rust so the suite depends on nothing but `cargo` and +//! `iroh-relay`: `keygen` writes a client key in the shared format, `fake-relay` +//! is a relay that answers net-report probes but refuses relay connections, +//! `delay-proxy` makes a relay measurably slower, and `pick-port` allocates +//! free localhost ports. + +use anyhow::{Context, Result, bail}; +use clap::{Args, Parser, Subcommand}; +use flexaccess_iroh::auth::{ClientKey, verify_endpoint_id_signature}; +use flexaccess_iroh::endpoint::{EndpointOptions, create_endpoint, endpoint_builder}; +use flexaccess_iroh::flexaccess_keys::{self, AuthorizedKeys, PrivateKey, PublicKey}; +use flexaccess_iroh::relay::RelayConfig; +use flexaccess_iroh::relay_failover::fail_over_home_relay; +use iroh::endpoint::{Connection, Incoming, QuicTransportConfig}; +use iroh::{Endpoint, EndpointAddr, EndpointId, SecretKey, TransportAddr}; +use log::{error, info, warn}; +use std::path::PathBuf; +use std::process::ExitCode; +use std::sync::Arc; +use std::time::{Duration, SystemTime}; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::{TcpListener, TcpStream}; + +/// The harness's own ALPN; no product speaks it. +const ALPN: &[u8] = b"flexaccess-iroh/e2e/1"; + +/// The harness's domain-separation context for the auth transcript. +const AUTH_CONTEXT: &[u8] = b"flexaccess-iroh-e2e-client-auth-v1"; + +/// Client exit status when the server rejects its authentication key. +const EXIT_AUTH_REJECTED: u8 = 3; + +/// Upper bound on one request or response; the protocol is two short lines. +const MAX_MESSAGE: usize = 64 * 1024; + +/// How long a server waits for a client to close after answering it. +const CLIENT_CLOSE_GRACE: Duration = Duration::from_secs(10); + +#[derive(Parser)] +#[command(about = "End-to-end harness for the flexaccess-iroh crate", version)] +struct Cli { + #[command(subcommand)] + command: Command, +} + +#[derive(Subcommand)] +enum Command { + /// Bind an endpoint, run the home-relay failover, and answer clients. + Server(ServerArgs), + /// Dial a server, prove the client key, and check one echo. + Client(ClientArgs), + /// Write a client authentication key in the shared flexaccess-keys format. + Keygen(KeygenArgs), + /// A relay that answers net-report probes (`GET /ping`) but refuses + /// relay connections (404 on everything else). + FakeRelay { + /// Port to listen on (127.0.0.1). + #[arg(long)] + port: u16, + }, + /// A TCP proxy that delays every new connection before forwarding it, + /// so the relay behind it always measures slower in the net report. + DelayProxy { + /// Port to listen on (127.0.0.1). + #[arg(long)] + listen: u16, + /// Port to forward to (127.0.0.1). + #[arg(long)] + upstream: u16, + /// Delay added before each new connection is forwarded. + #[arg(long)] + delay_ms: u64, + }, + /// Print free localhost TCP ports, one per line, all distinct. + PickPort { + #[arg(long, default_value_t = 1)] + count: usize, + }, +} + +/// The relay choice shared by `server` and `client`, mapped straight onto +/// [`RelayConfig`] and [`EndpointOptions`]. +#[derive(Args)] +struct RelayArgs { + /// Custom relay URL (repeatable; at least two distinct, or none for the + /// default relays). Custom relays disable n0 internet discovery. + #[arg(long = "relay-url")] + relay_urls: Vec, + /// Shared relay auth token, sent to every custom relay. + #[arg(long, env = "E2E_RELAY_AUTH_TOKEN")] + relay_auth_token: Option, + /// Reach peers only through the configured relays: no direct paths, no + /// address lookup of any kind. Requires custom relays. + #[arg(long)] + relay_only: bool, +} + +impl RelayArgs { + fn resolve(&self) -> Result { + let config = RelayConfig::from_urls_with_token(&self.relay_urls, self.relay_auth_token.clone())?; + if self.relay_only && !config.is_custom() { + bail!("--relay-only requires custom relays (--relay-url, at least two)"); + } + Ok(config) + } + + fn options(&self, publish_address: bool) -> EndpointOptions { + EndpointOptions { + transport_config: QuicTransportConfig::default(), + publish_address, + relay_only: self.relay_only, + } + } +} + +#[derive(Args)] +struct ServerArgs { + #[command(flatten)] + relay: RelayArgs, + /// Authorized-keys file (flexaccess-keys format). + #[arg(long)] + authorized_keys: PathBuf, + /// The server identity as a base64 32-byte iroh secret key. A fresh + /// identity is generated when absent; either way the EndpointId is logged. + #[arg(long, env = "E2E_SERVER_SECRET", hide_env_values = true)] + secret: Option, +} + +#[derive(Args)] +struct ClientArgs { + #[command(flatten)] + relay: RelayArgs, + /// The server's EndpointId, as logged by `server`. + #[arg(long)] + server_id: EndpointId, + /// Client authentication key file (flexaccess-keys format). + #[arg(long)] + private_key_file: PathBuf, + /// Payload the server must echo back (a single line). + #[arg(long, default_value = "hello")] + message: String, + /// Seconds to wait for the connection to the server. + #[arg(long, default_value_t = 30)] + connect_timeout: u64, +} + +#[derive(Args)] +struct KeygenArgs { + /// Authorized-key comment naming the client. + comment: String, + /// Where to write the private key file (created with mode 0600). + #[arg(long)] + private_key_file: PathBuf, + /// Append the key's authorized-keys entry to this file. + #[arg(long)] + authorized_keys: Option, +} + +#[tokio::main] +async fn main() -> ExitCode { + // The scripts assert on this crate's and the harness's own lines; iroh's + // tracing (bridged into `log`, spans included) is opt-in via RUST_LOG. + env_logger::Builder::from_env( + env_logger::Env::default().default_filter_or("info,iroh=warn,iroh_relay=warn,tracing=off"), + ) + .init(); + let cli = Cli::parse(); + match run(cli.command).await { + Ok(code) => code, + Err(e) => { + error!("{e:#}"); + ExitCode::FAILURE + } + } +} + +async fn run(command: Command) -> Result { + match command { + Command::Server(args) => run_server(args).await.map(|()| ExitCode::SUCCESS), + Command::Client(args) => run_client(args).await, + Command::Keygen(args) => keygen(args).map(|()| ExitCode::SUCCESS), + Command::FakeRelay { port } => fake_relay(port).await.map(|()| ExitCode::SUCCESS), + Command::DelayProxy { + listen, + upstream, + delay_ms, + } => delay_proxy(listen, upstream, Duration::from_millis(delay_ms)) + .await + .map(|()| ExitCode::SUCCESS), + Command::PickPort { count } => pick_ports(count).map(|()| ExitCode::SUCCESS), + } +} + +// --------------------------------------------------------------------------- +// server +// --------------------------------------------------------------------------- + +async fn run_server(args: ServerArgs) -> Result<()> { + let authorized = flexaccess_keys::load_authorized_keys(&args.authorized_keys) + .map_err(anyhow::Error::from)?; + if authorized.is_empty() { + warn!("{} holds no keys; every client will be rejected", args.authorized_keys.display()); + } + let relay_config = args.relay.resolve()?; + let secret = match &args.secret { + Some(secret) => parse_secret(secret)?, + None => SecretKey::generate(), + }; + info!("EndpointId: {}", secret.public()); + + let builder = endpoint_builder(&relay_config, args.relay.options(true)) + .alpns(vec![ALPN.to_vec()]) + .secret_key(secret); + let endpoint = create_endpoint(&relay_config, builder).await?; + info!("Waiting for clients to connect"); + + let authorized = Arc::new(authorized); + let outcome = tokio::select! { + outcome = accept_loop(&endpoint, &authorized) => outcome, + () = fail_over_home_relay(&endpoint, &relay_config) => Ok(()), + }; + endpoint.close().await; + outcome +} + +/// A base64 32-byte secret key. Any 32 bytes are a valid Ed25519 seed, so a +/// script can mint one with `head -c 32 /dev/urandom | base64`. +fn parse_secret(encoded: &str) -> Result { + use base64::Engine; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded.trim()) + .context("--secret is not valid base64")?; + SecretKey::try_from(&bytes[..]).context("--secret must decode to 32 bytes") +} + +async fn accept_loop(endpoint: &Endpoint, authorized: &Arc) -> Result<()> { + while let Some(incoming) = endpoint.accept().await { + let authorized = Arc::clone(authorized); + tokio::spawn(async move { + if let Err(e) = serve(incoming, &authorized).await { + warn!("Connection failed: {e:#}"); + } + }); + } + bail!("the endpoint stopped accepting connections") +} + +/// One request per connection: the auth line, then the message to echo. +async fn serve(incoming: Incoming, authorized: &AuthorizedKeys) -> Result<()> { + let conn: Connection = incoming.await.context("accepting connection")?; + let remote = conn.remote_id(); + let (mut send, mut recv) = conn.accept_bi().await.context("accepting stream")?; + let request = recv + .read_to_end(MAX_MESSAGE) + .await + .context("reading request")?; + let request = String::from_utf8(request).context("request is not UTF-8")?; + let mut lines = request.lines(); + let auth_line = lines.next().context("request without an auth line")?; + let message = lines.next().unwrap_or_default(); + + let response = match authenticate(auth_line, &remote, authorized) { + Ok(comment) => { + info!("Client {remote} authenticated successfully as {comment}"); + format!("OK {comment}\n{message}\n") + } + Err(reason) => { + // The client learns only that its proof failed, never which check. + warn!("Rejected client {remote}: {reason}"); + "REJECTED Invalid authentication proof\n".to_string() + } + }; + send.write_all(response.as_bytes()) + .await + .context("writing response")?; + send.finish().context("finishing response")?; + // The client closes once it has read the response; give it a moment so + // the response is not lost to an early close from this side. + let _ = tokio::time::timeout(CLIENT_CLOSE_GRACE, conn.closed()).await; + Ok(()) +} + +/// Check ` ` against the connection's +/// TLS-authenticated remote id and the authorized keys. `Ok` is the key's +/// comment; `Err` says why it was refused. +fn authenticate( + line: &str, + remote: &EndpointId, + authorized: &AuthorizedKeys, +) -> std::result::Result { + let mut fields = line.split_whitespace(); + let (Some(public), Some(claimed), Some(signature), None) = + (fields.next(), fields.next(), fields.next(), fields.next()) + else { + return Err("malformed auth line".into()); + }; + let public: PublicKey = public + .parse() + .map_err(|e| format!("invalid public key: {e}"))?; + let claimed: EndpointId = claimed + .parse() + .map_err(|e| format!("invalid endpoint id: {e}"))?; + if claimed != *remote { + return Err(format!("claimed endpoint id {claimed} is not the connection's {remote}")); + } + if !verify_endpoint_id_signature(&public, AUTH_CONTEXT, remote, signature) { + return Err(format!("invalid signature for {public}")); + } + match authorized.comment(&public) { + Some(comment) => Ok(comment.to_string()), + None => Err(format!("key {public} is not authorized")), + } +} + +// --------------------------------------------------------------------------- +// client +// --------------------------------------------------------------------------- + +async fn run_client(args: ClientArgs) -> Result { + if args.message.contains(['\n', '\r']) { + bail!("--message must be a single line"); + } + let key: ClientKey = flexaccess_keys::load_private_key(&args.private_key_file) + .map_err(anyhow::Error::from)? + .into(); + let relay_config = args.relay.resolve()?; + let builder = endpoint_builder(&relay_config, args.relay.options(false)); + let endpoint = create_endpoint(&relay_config, builder).await?; + let outcome = exchange(&endpoint, &args, &relay_config, &key).await; + endpoint.close().await; + outcome +} + +async fn exchange( + endpoint: &Endpoint, + args: &ClientArgs, + relay_config: &RelayConfig, + key: &ClientKey, +) -> Result { + // With custom relays discovery is off, so the relay hints are how the + // server is reached at all: iroh sends the handshake through every one of + // them and it succeeds via whichever relay the server is homed on. + let mut addr = EndpointAddr::new(args.server_id); + for url in relay_config.custom_urls() { + addr = addr.with_relay_url(url.clone()); + } + let timeout = Duration::from_secs(args.connect_timeout); + info!( + "Connecting to {} with {} relay hint(s) (timeout {}s)...", + args.server_id, + relay_config.custom_urls().len(), + timeout.as_secs() + ); + let conn = tokio::time::timeout(timeout, endpoint.connect(addr, ALPN)) + .await + .map_err(|_| anyhow::anyhow!("connection to {} timed out after {}s", args.server_id, timeout.as_secs()))? + .context("connecting to the server")?; + info!("Connected to {} via {}", args.server_id, describe_paths(&conn)); + + let id = endpoint.id(); + let signature = key.sign_endpoint_id(AUTH_CONTEXT, &id); + let request = format!("{} {id} {signature}\n{}\n", key.public_str(), args.message); + let (mut send, mut recv) = conn.open_bi().await.context("opening stream")?; + send.write_all(request.as_bytes()) + .await + .context("sending request")?; + send.finish().context("finishing request")?; + let response = recv + .read_to_end(MAX_MESSAGE) + .await + .context("reading response")?; + conn.close(0u32.into(), b"done"); + let response = String::from_utf8(response).context("response is not UTF-8")?; + + let mut lines = response.lines(); + let status = lines.next().context("empty response")?; + if let Some(reason) = status.strip_prefix("REJECTED ") { + error!("Authentication rejected: {reason}"); + return Ok(ExitCode::from(EXIT_AUTH_REJECTED)); + } + let comment = status + .strip_prefix("OK ") + .with_context(|| format!("unexpected response status: {status:?}"))?; + info!("Authenticated as {comment}"); + let echoed = lines.next().unwrap_or_default(); + if echoed != args.message { + bail!("echo mismatch: sent {:?}, got {echoed:?}", args.message); + } + info!("Echo OK ({} bytes)", args.message.len()); + Ok(ExitCode::SUCCESS) +} + +/// The selected paths of a connection, e.g. `Relay http://127.0.0.1:3340/` +/// or `Direct 127.0.0.1:41234`. +fn describe_paths(conn: &Connection) -> String { + let paths = conn.paths(); + let selected: Vec = paths + .iter() + .filter(|path| path.is_selected()) + .map(|path| match path.remote_addr() { + TransportAddr::Relay(url) => format!("Relay {url}"), + TransportAddr::Ip(addr) => format!("Direct {addr}"), + other => format!("{other:?}"), + }) + .collect(); + if selected.is_empty() { + "no selected path yet".to_string() + } else { + selected.join(", ") + } +} + +// --------------------------------------------------------------------------- +// fixtures +// --------------------------------------------------------------------------- + +fn keygen(args: KeygenArgs) -> Result<()> { + let key = PrivateKey::generate().map_err(anyhow::Error::from)?; + let key_file = key + .to_key_file(&args.comment, SystemTime::now()) + .map_err(anyhow::Error::from)?; + flexaccess_keys::write_private_key_file(&args.private_key_file, &key_file, false) + .map_err(anyhow::Error::from)?; + let entry = key + .authorized_key(&args.comment) + .map_err(anyhow::Error::from)?; + if let Some(path) = &args.authorized_keys { + let mut file = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(path) + .with_context(|| format!("opening {}", path.display()))?; + use std::io::Write; + writeln!(file, "{entry}")?; + } + println!("{entry}"); + Ok(()) +} + +async fn fake_relay(port: u16) -> Result<()> { + let listener = TcpListener::bind(("127.0.0.1", port)) + .await + .with_context(|| format!("binding 127.0.0.1:{port}"))?; + println!("READY fake relay on 127.0.0.1:{port}"); + loop { + let (stream, _) = listener.accept().await?; + tokio::spawn(async move { + let _ = serve_fake_relay(stream).await; + }); + } +} + +/// Minimal HTTP/1.1: `GET /ping` gets a 200, anything else (in particular +/// the `/relay` WebSocket upgrade) a 404. Keep-alive, no request bodies. +async fn serve_fake_relay(mut stream: TcpStream) -> Result<()> { + let mut buf = Vec::new(); + let mut chunk = [0u8; 4096]; + loop { + let n = stream.read(&mut chunk).await?; + if n == 0 { + return Ok(()); + } + buf.extend_from_slice(&chunk[..n]); + while let Some(end) = buf.windows(4).position(|w| w == b"\r\n\r\n") { + let head: Vec = buf.drain(..end + 4).collect(); + let request_line = head.split(|&b| b == b'\r').next().unwrap_or_default(); + let response: &[u8] = if request_line.starts_with(b"GET /ping ") { + b"HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\nContent-Length: 4\r\n\r\npong" + } else { + b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n" + }; + stream.write_all(response).await?; + } + } +} + +async fn delay_proxy(listen: u16, upstream: u16, delay: Duration) -> Result<()> { + let listener = TcpListener::bind(("127.0.0.1", listen)) + .await + .with_context(|| format!("binding 127.0.0.1:{listen}"))?; + println!( + "READY delay proxy 127.0.0.1:{listen} -> 127.0.0.1:{upstream} (+{}ms)", + delay.as_millis() + ); + loop { + let (mut client, _) = listener.accept().await?; + tokio::spawn(async move { + tokio::time::sleep(delay).await; + // Established connections are forwarded byte for byte: the relay + // behind the proxy works normally, it is just never the fastest. + if let Ok(mut server) = TcpStream::connect(("127.0.0.1", upstream)).await { + let _ = tokio::io::copy_bidirectional(&mut client, &mut server).await; + } + }); + } +} + +/// Hold every port bound until all are chosen so they are distinct. +fn pick_ports(count: usize) -> Result<()> { + let listeners: Vec = (0..count) + .map(|_| std::net::TcpListener::bind("127.0.0.1:0")) + .collect::>() + .context("binding an ephemeral port")?; + for listener in &listeners { + println!("{}", listener.local_addr()?.port()); + } + Ok(()) +} From cd490d1395fc4cf8f314dd60434057edcf60fd8f Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Fri, 4 Sep 2026 21:46:04 -0700 Subject: [PATCH 2/2] Tighten the e2e negative check and make cleanup bounded and single-shot A server that died without logging the expected failure counted as the expected failure; now only the logged failure passes. The cleanup trap clears itself on entry so an INT/TERM exit cannot run it twice, and it SIGKILLs children that ignore SIGTERM after a short grace period instead of waiting on them forever. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_0163AtLdT9AWeDE3EYmhFYXK --- e2e/lib.sh | 26 ++++++++++++++++++++++++-- e2e/run_relay_failover.sh | 8 ++++---- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/e2e/lib.sh b/e2e/lib.sh index 2732663..c60e41f 100644 --- a/e2e/lib.sh +++ b/e2e/lib.sh @@ -46,10 +46,23 @@ declare -a PIDS=() cleanup() { local status=$? - for pid in "${PIDS[@]:-}"; do - [[ -n "$pid" ]] || continue + # Runs once: an INT/TERM handler that exits would fire the EXIT trap again. + trap - EXIT INT TERM + local pid + for pid in ${PIDS[@]+"${PIDS[@]}"}; do kill -TERM -- "-$pid" 2>/dev/null || kill -TERM "$pid" 2>/dev/null || true done + # A short grace period for SIGTERM, then SIGKILL whatever is left, so a + # child that ignores SIGTERM cannot hang the run. + local _ + for _ in $(seq 1 20); do + live_children || break + sleep 0.1 + done + for pid in ${PIDS[@]+"${PIDS[@]}"}; do + kill -0 "$pid" 2>/dev/null || continue + kill -KILL -- "-$pid" 2>/dev/null || kill -KILL "$pid" 2>/dev/null || true + done wait 2>/dev/null || true if [[ "${KEEP_LOGS:-0}" == "1" ]]; then echo "==> Logs kept in $WORK" @@ -60,6 +73,15 @@ cleanup() { } trap cleanup EXIT INT TERM +# True while any process started with start_bg is still alive. +live_children() { + local pid + for pid in ${PIDS[@]+"${PIDS[@]}"}; do + kill -0 "$pid" 2>/dev/null && return 0 + done + return 1 +} + # Start a background process, in its own session where setsid exists so the # whole process group can be killed. Args: . Records # the PID in PIDS and BG_PID. diff --git a/e2e/run_relay_failover.sh b/e2e/run_relay_failover.sh index 89e1c2e..de8bab3 100755 --- a/e2e/run_relay_failover.sh +++ b/e2e/run_relay_failover.sh @@ -158,13 +158,13 @@ expect_server_start_failure() { local pattern="$1"; shift local rc=0 start_server "$@" + # rc 0: the expected failure was logged (also when the process exited + # right after logging it). rc 1: timed out. rc 2: the process died without + # logging it, i.e. it failed for some other reason. Only rc 0 passes, and + # only if the server never became ready. wait_for_log_or_death "$SERVER_PID" "$SERVER_LOG" "$pattern" "$READY_TIMEOUT" || rc=$? - # rc 0 (error logged) or 2 (process died after logging) both mean it gave - # up; verify it never became ready. if grep -Eq "Waiting for clients to connect" "$SERVER_LOG"; then rc=1 - elif [[ "$rc" -eq 2 ]]; then - rc=0 fi [[ "$rc" -eq 0 ]] || dump_log "$SERVER_LOG" stop_server