diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d2e9e3..4ea5870 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,8 +57,9 @@ jobs: # The end-to-end suites that need no public infrastructure: the auth # transcript and connectivity over two local iroh-relay instances, and the - # relay failover scenarios (which take ~6 minutes by design: a 60 s outage - # window and a 90 s restore probe). See e2e/README.md. + # relay failover scenarios (which take ~8 minutes by design: a 60 s outage + # window and a 90 s restore probe relay-only, then another 60 s window with + # direct paths allowed). See e2e/README.md. e2e: name: e2e (local relays) runs-on: ubuntu-latest diff --git a/AGENTS.md b/AGENTS.md index cfc71ab..54d97d8 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,3 +1,3 @@ -- strict no backward compatibility +- strict no backward compatibility or legacy code paths - run cargo clippy and test after rust code changes - no cargo fmt \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 3532ea2..823d171 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -843,7 +843,7 @@ checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "flexaccess-iroh" -version = "0.0.7" +version = "0.0.8" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index 0e0d9b8..df0172f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "flexaccess-iroh" -version = "0.0.7" +version = "0.0.8" edition = "2024" description = "Shared iroh transport layer for FlexAccess applications: relay configuration and probing, endpoint building, in-place home-relay failover, and the endpoint-bound public-key auth transcript" repository = "https://github.com/flexaccessdev/flexaccess-iroh" diff --git a/README.md b/README.md index a2ef398..4b4f9bd 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,9 @@ hand into every repo. | Module | Contents | |---|---| -| `relay` | `RelayConfig` (default vs custom relays, which also decides whether n0 internet discovery is on; custom relays must number at least two distinct URLs; duplicates are collapsed before the count), the shared relay auth token, the per-relay startup probe (fails only when no relay is reachable) | -| `endpoint` | the common endpoint builder and `create_endpoint` | -| `relay_failover` | the server-side home-relay failover: after 60 s without a connected home relay, take the wedged relay out of the relay map so the forced net report homes the endpoint on another configured relay, in place; put it back once a probe shows it connectable | +| `relay` | `RelayConfig` (default vs custom relays, which also decides whether n0 internet discovery is on; custom relays must number at least two distinct URLs; duplicates are collapsed before the count), the shared relay auth token, the per-relay startup probe (fails only when no relay is reachable, and reports which relays did not come online) | +| `endpoint` | the common endpoint builder and `create_endpoint`, which binds **without** the relays that failed the startup probe (a relay that answers probes but cannot be connected would otherwise be preferred as home relay and keep the endpoint from ever coming online, so every client restarting during such an outage would fail to start) and hands them back for the failover to restore | +| `relay_failover` | the server-side home-relay failover: after 60 s without a connected home relay, take the wedged relay out of the relay map so the forced net report homes the endpoint on another configured relay, in place; put it back once a probe shows it connectable (likewise for relays left out at startup) | | `auth` | the endpoint-bound public-key auth transcript over the [flexaccess-keys] format; each application passes its own domain-separation context | Deliberately **not** in it: ALPNs, handshake wire formats, QUIC transport @@ -35,9 +35,9 @@ takes the resulting `iroh::SecretKey` / `flexaccess_keys` values. ```toml [dependencies] -flexaccess-iroh = { git = "https://github.com/flexaccessdev/flexaccess-iroh", tag = "v0.0.7" } +flexaccess-iroh = { git = "https://github.com/flexaccessdev/flexaccess-iroh", tag = "v0.0.8" } # or, with mDNS local-network discovery on every endpoint (compiled out on iOS): -flexaccess-iroh = { git = "...", tag = "v0.0.7", features = ["mdns"] } +flexaccess-iroh = { git = "...", tag = "v0.0.8", features = ["mdns"] } ``` The `flexaccess_keys` crate is re-exported so a consumer signs and verifies diff --git a/e2e/README.md b/e2e/README.md index d7920d1..3cfd501 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -14,8 +14,8 @@ through real relays — deliberately not an application. Built with | Subcommand | Role | |---|---| -| `server` | binds an endpoint with the shared builder (`--relay-url`, `--relay-only`, an optional `E2E_SERVER_SECRET` identity), runs `relay_failover::fail_over_home_relay` beside its accept loop, and answers one request per connection: the endpoint-bound auth transcript from `flexaccess_iroh::auth` against `--authorized-keys`, then an echo of the client's message. Logs `EndpointId: …` and `Waiting for clients to connect` when ready. | -| `client` | builds an ephemeral endpoint the same way, dials `--server-id` through the configured relays (every custom relay as a hint), proves `--private-key-file`, checks the echo. Exits `0` on `Echo OK`, `3` when the server rejects the key, `1` otherwise. Logs the path it connected over (`via Relay ` / `via Direct `). | +| `server` | binds an endpoint with the shared builder (`--relay-url`, `--relay-only`, an optional `E2E_SERVER_SECRET` identity), runs `relay_failover::fail_over_home_relay` beside its accept loop, and answers each connection: the endpoint-bound auth transcript from `flexaccess_iroh::auth` against `--authorized-keys`, then an echo of the client's message, then an echo of every further stream the client opens. Logs `EndpointId: …` and `Waiting for clients to connect` when ready. | +| `client` | builds an ephemeral endpoint the same way, dials `--server-id` through the configured relays (every custom relay as a hint), proves `--private-key-file`, checks the echo. Exits `0` on `Echo OK`, `3` when the server rejects the key, `1` otherwise. Logs the selected path it connected over (`via Relay ` / `via Direct `) followed by every path of the connection with its RTT and whether it is selected (`[paths: …]`), the same view flextunnel's connection-path status shows. With `--hold-secs N` it then keeps the connection open, echoing every `--echo-interval-secs` on a fresh stream and logging `Echo #n OK via [paths: …]`, so a script can watch a live connection's paths through a relay outage. | | `keygen` | writes a client key in the shared flexaccess-keys format (mode 0600) and its `authorized_keys` entry | | `fake-relay` | a relay that answers the net-report probe (`GET /ping`) but refuses relay connections — the outage shape the failover exists for | | `delay-proxy` | a TCP proxy adding latency to each new connection, so the relay behind it always measures slower | @@ -24,12 +24,18 @@ through real relays — deliberately not an application. Built with Everything a product adds — its ALPN, QUIC tuning, config files, forwarding — is left out, so a failure here is a failure of this crate or of iroh. +The harness is built **without** the crate's `mdns` feature, and refuses to +run with direct paths allowed if it was built with it: every process here is +on one host, so mDNS would hand a client the server's address directly and +the relays would never be exercised. Without it, the handshake goes through +the relay and a direct path is learned through it, exactly as off-LAN. + ## Suites | Script | What it checks | Needs | |---|---|---| | `run_e2e.sh` | **Auth + connectivity.** An unlisted key is rejected (exit 3, explicit rejection, server logs why); an authorized client authenticates and gets its echo; a second client on the *same* key uses a *distinct* ephemeral iroh identity; under `--relay-only`, every connection ran through a relay. | internet for the default relays; or `--local-relays` / `--relay-url` ×2 for a fully offline run | -| `run_relay_failover.sh` | **Relay failover**, fully offline against two local `iroh-relay --dev` instances, relay-only. Phase A: relays down *before* startup (per-relay probe: startup fails only when none is reachable; a single custom relay is rejected). Phase B: a relay dies *after* startup (iroh re-homes on its own; both down fails new clients; both back recovers). Phase C: the home relay answers probes but refuses connections (the in-place failover removes it after 60 s, the server homes on the other relay without restarting, and the restore probe puts it back once it is connectable again). | `iroh-relay` | +| `run_relay_failover.sh` | **Relay failover**, fully offline against two local `iroh-relay --dev` instances (relay-only until phase D). Phase A: relays down *before* startup (per-relay probe: startup fails only when none is reachable; a single custom relay is rejected). Phase B: a relay dies *after* startup (iroh re-homes on its own; both down fails new clients; both back recovers). Phase C: the home relay answers probes but refuses connections (the in-place failover removes it after 60 s, the server homes on the other relay without restarting, and the restore probe puts it back once it is connectable again). Phase D: the same wedge with direct paths allowed and a client holding a live connection whose paths are relay1 + direct; the failover still moves the server onto the other relay in the background, the held connection keeps echoing over its direct path (ending up direct-only: iroh drops the wedged relay path and does not add a peer's new home relay to an established connection), a new relay-only client's connection path shows the new home relay, and a client that restarts with the same two-relay configuration during the outage comes online, reaches the server through the new home relay, and goes direct again. | `iroh-relay` | ```sh cargo install iroh-relay --features server # one-time @@ -39,7 +45,7 @@ cargo install iroh-relay --features server # one-time ./e2e/run_e2e.sh --local-relays --relay-only # ... relay paths only ./e2e/run_e2e.sh --relay-url URL --relay-url URL [--relay-only] # your own relays -./e2e/run_relay_failover.sh # ~6 minutes: 60 s failover window + 90 s restore probe +./e2e/run_relay_failover.sh # ~8 minutes: two 60 s failover windows + a 90 s restore probe ``` Each suite prints a `PASS`/`FAIL` line per scenario and exits non-zero on diff --git a/e2e/run_relay_failover.sh b/e2e/run_relay_failover.sh index de8bab3..ced2c0a 100755 --- a/e2e/run_relay_failover.sh +++ b/e2e/run_relay_failover.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Relay failover end-to-end test for flexaccess-iroh (relay-only, no internet). +# Relay failover end-to-end test for flexaccess-iroh (no internet). # # Runs TWO local iroh-relay instances (`--dev` mode, plain HTTP) and exercises # relay failures against the e2e harness (examples/e2e). Servers and clients @@ -44,10 +44,37 @@ # it back in the relay map (checked every 90s), the server moves back onto # it, and a client connects via relay1; echo passes # +# Phase D - direct paths allowed (NOT relay-only) while the home relay wedges: +# the failover is about relay reachability and must run the same +# behind a live direct connection, which is what masked the original +# incident (LAN clients kept working while nothing off-LAN could dial). +# The harness is built without the crate's `mdns` feature (and refuses +# mixed mode otherwise): the handshake has to go through the relay and +# the direct path is learned through it, exactly as off-LAN; with mDNS +# the client would dial the server directly and skip the relays. +# D0 same relay setup as C0, server and a long-lived client with direct paths +# allowed; the server homes on relay1; the client connects through +# relay1, holds the connection open and keeps echoing, logging every +# path of the connection; its echoes move onto a direct path while the +# relay1 path stays beside it (relay1 + direct) +# D1 relay1 is replaced by the fake as in C1; the held client's echoes keep +# flowing over the direct path throughout; the failover still removes +# relay1 after 60s and homes the server on relay2 without a restart; the +# held connection ends up direct-only (the wedged relay1 path is gone; +# iroh does not add a peer's new home relay to an established +# connection); a new relay-only client connects via relay2, which is what +# shows the new home relay on a connection path; echo passes +# D2 the held client disconnects and a client with the SAME two-relay +# configuration (relay1 still the fake, direct paths allowed) dials +# again, as a LAN client that restarts during the outage would: its +# startup probe finds relay1 not connectable and it binds without it, so +# it comes online on relay2, reaches the re-homed server through relay2, +# and moves onto a direct path again; echo passes +# # Requirements: cargo, iroh-relay (cargo install iroh-relay --features server). # # Usage: -# ./e2e/run_relay_failover.sh +# ./e2e/run_relay_failover.sh # ~8 minutes # # Environment: E2E_BIN, IROH_RELAY_BIN, KEEP_LOGS, READY_TIMEOUT (see lib.sh). # @@ -141,13 +168,18 @@ relay_args() { done } -# Start the server in relay-only mode. Args: ... +# Whether the server is started relay-only (phases A-C) or with direct paths +# allowed (phase D). +SERVER_RELAY_ONLY=1 + +# Start the server. Args: ... start_server() { SERVER_LOG="$WORK/server.$(date +%s%N).log" local -a args=() mapfile -d '' -t args < <(relay_args "$@") + [[ "$SERVER_RELAY_ONLY" -eq 1 ]] && args+=(--relay-only) E2E_SERVER_SECRET="$SECRET" start_bg "$SERVER_LOG" \ - "$BIN" server --relay-only --authorized-keys "$WORK/authorized_keys" "${args[@]}" + "$BIN" server --authorized-keys "$WORK/authorized_keys" "${args[@]}" SERVER_PID="$BG_PID" } @@ -250,6 +282,62 @@ connect_and_echo() { return 1 } +# connect_and_echo, then require that the connection's path ran through the +# relay at $1: the client logs the path it connected over, and a relay path +# names the server's home relay, so this is what confirms a (re-)home from +# the outside rather than just "some relay worked". +# Args: ... +connect_and_echo_via() { + local expected="$1"; shift + connect_and_echo "$@" || return 1 + if ! grep -Eq "Connected to [0-9a-f]+ via Relay $expected/" "$CLIENT_LOG"; then + note "client did not connect through $expected" + dump_log "$CLIENT_LOG" + return 1 + fi +} + +# A long-lived client with direct paths allowed (phase D): connects, then +# holds the connection open, echoing every few seconds on a fresh stream and +# logging the path of each echo ("Echo #n OK via Direct ..."). Killed by +# stop_held_client once the scenario has seen enough. Args: ... +HELD_PID="" +HELD_LOG="" +HELD_ECHO_INTERVAL=5 +start_held_client() { + HELD_LOG="$WORK/held_client.$(date +%s%N).log" + local -a args=() + mapfile -d '' -t args < <(relay_args "$@") + start_bg "$HELD_LOG" "$BIN" client --server-id "$SERVER_ID" \ + --private-key-file "$WORK/client.key" --message "held-$(date +%s%N)" \ + --hold-secs 600 --echo-interval-secs "$HELD_ECHO_INTERVAL" "${args[@]}" + HELD_PID="$BG_PID" +} + +stop_held_client() { + kill_pid "$HELD_PID" + HELD_PID="" +} + +# How many held echoes have completed so far. +held_echo_count() { + grep -c "Echo #[0-9]* OK via" "$HELD_LOG" || true +} + +# Wait until the held client has completed more than $1 echoes, or time out +# after $2 seconds. Fails early if the client process dies. +wait_for_held_echoes_past() { + local count="$1" timeout="$2" + local max_attempts=$(( timeout * 2 )) attempt=0 + while (( attempt < max_attempts )); do + (( $(held_echo_count) > count )) && return 0 + kill -0 "$HELD_PID" 2>/dev/null || return 2 + sleep 0.5 + attempt=$(( attempt + 1 )) + done + return 1 +} + # The message the startup relay probe emits when EVERY configured relay is # down (one dead relay is only a warning). Negative scenarios must fail in # the expected way; requiring the message keeps an unrelated startup failure @@ -316,16 +404,12 @@ if [[ "$rc" -eq 0 ]] && ! grep -Eq "1 of 2 custom relays failed to come online" rc=1 fi if [[ "$rc" -eq 0 ]]; then - connect_and_echo 3 "$RELAY1_URL" "$RELAY2_URL" || rc=1 + connect_and_echo_via "$RELAY2_URL" 3 "$RELAY1_URL" "$RELAY2_URL" || rc=1 fi if [[ "$rc" -eq 0 ]] && ! grep -Eq "1 of 2 custom relays failed to come online" "$CLIENT_LOG"; then note "client did not warn about the dead relay" rc=1 fi -if [[ "$rc" -eq 0 ]] && ! grep -Eq "Connected to [0-9a-f]+ via Relay $RELAY2_URL/" "$CLIENT_LOG"; then - note "client did not connect through relay2" - rc=1 -fi record A1 "$rc" stop_server @@ -380,9 +464,10 @@ if [[ -n "$HOME_RELAY_NUM" ]]; then note "the failover acted although iroh re-homed on its own" rc=1 fi - # The new client lists both relays (one is dead: a warning at its probe). + # The new client lists both relays (one is dead: a warning at its probe) + # and must come in through the survivor. if [[ "$rc" -eq 0 ]]; then - connect_and_echo 4 "$RELAY1_URL" "$RELAY2_URL" || rc=1 + connect_and_echo_via "http://127.0.0.1:$SURVIVOR_PORT" 4 "$RELAY1_URL" "$RELAY2_URL" || rc=1 fi else rc=1 @@ -425,7 +510,7 @@ if [[ "$rc" -eq 0 ]]; then wait_for_home_relay "$RELAY1_PORT" 30 || { note "server did not home on relay1"; rc=1; } fi if [[ "$rc" -eq 0 ]]; then - connect_and_echo 3 "$RELAY1_URL" "$PROXY_URL" || rc=1 + connect_and_echo_via "$RELAY1_URL" 3 "$RELAY1_URL" "$PROXY_URL" || rc=1 fi record C0 "$rc" @@ -460,7 +545,7 @@ fi # dead port rather than the fake: a client that homed on the fake could never # come online, and the failover runs only on the server. if [[ "$rc" -eq 0 ]]; then - connect_and_echo 3 "$PROXY_URL" "$DEAD_URL" || rc=1 + connect_and_echo_via "$PROXY_URL" 3 "$PROXY_URL" "$DEAD_URL" || rc=1 fi [[ "$rc" -eq 0 ]] || dump_log "$SERVER_LOG" record C1 "$rc" @@ -477,7 +562,7 @@ if [[ "$rc" -eq 0 ]]; then wait_for_home_relay "$RELAY1_PORT" 60 || { note "server did not move back onto relay1"; rc=1; } fi if [[ "$rc" -eq 0 ]]; then - connect_and_echo 3 "$RELAY1_URL" "$PROXY_URL" || rc=1 + connect_and_echo_via "$RELAY1_URL" 3 "$RELAY1_URL" "$PROXY_URL" || rc=1 fi [[ "$rc" -eq 0 ]] || dump_log "$SERVER_LOG" record C2 "$rc" @@ -487,6 +572,155 @@ stop_delay_proxy stop_relay 1 stop_relay 2 +# =========================================================================== +# Phase D - direct paths allowed: the failover still runs behind a live direct +# connection +# =========================================================================== + +SERVER_RELAY_ONLY=0 + +scenario D0 "direct paths allowed: server homes on relay1, a held client's echoes run over a direct path" +rc=0 +start_relay 1 +start_relay 2 +start_delay_proxy +expect_server_ready "$RELAY1_URL" "$PROXY_URL" || rc=1 +if [[ "$rc" -eq 0 ]]; then + wait_for_home_relay "$RELAY1_PORT" 30 || { note "server did not home on relay1"; rc=1; } +fi +if [[ "$rc" -eq 0 ]]; then + start_held_client "$RELAY1_URL" "$PROXY_URL" + wait_for_log_or_death "$HELD_PID" "$HELD_LOG" "Echo OK" "$READY_TIMEOUT" || { + note "the held client did not connect"; rc=1; } +fi +# The handshake runs through the relay; the direct path is added once the +# endpoints have exchanged addresses through it and is preferred from then on. +if [[ "$rc" -eq 0 ]]; then + wait_for_log_or_death "$HELD_PID" "$HELD_LOG" "Echo #[0-9]+ OK via Direct " 30 || { + note "the held client's echoes never moved onto a direct path"; rc=1; } +fi +# Mixed mode proper: the connection holds both the relay1 path it was set up +# through and the direct path it now uses (mDNS would have skipped relay1). +if [[ "$rc" -eq 0 ]]; then + held_paths="$(grep -E "Echo #[0-9]+ OK via Direct " "$HELD_LOG" | head -1 | grep -Eo "\[paths: .*\]")" + note "held connection paths: $held_paths" + if ! grep -q "Relay $RELAY1_URL/" <<<"$held_paths"; then + note "the held connection has no relay1 path beside the direct one" + rc=1 + fi +fi +[[ "$rc" -eq 0 ]] || { dump_log "$SERVER_LOG"; [[ -n "$HELD_LOG" ]] && dump_log "$HELD_LOG"; } +record D0 "$rc" + +scenario D1 "relay1 becomes the fake while the direct connection is live: the failover still moves the server onto relay2, the held client never notices" +rc=0 +stop_relay 1 +start_fake_relay +wait_for_log "$SERVER_LOG" "No connected home relay" 60 || { note "server never noticed the relay loss"; rc=1; } +if [[ "$rc" -eq 0 ]]; then + wait_for_log "$SERVER_LOG" "Removed $RELAY1_URL/ from the relay map" 90 || { + note "the failover did not remove relay1 from the relay map"; rc=1; } +fi +# Echoes completed by the time the relay map changed: the ones after it are +# the ones that prove the direct connection survived the failover. +ECHOES_AT_REMOVAL="$(held_echo_count)" +if [[ "$rc" -eq 0 ]]; then + wait_for_log "$SERVER_LOG" "Home relay connection restored on $PROXY_URL/" 60 || { + note "server did not home on relay2 after the failover"; rc=1; } +fi +if [[ "$rc" -eq 0 ]]; then + removed_line="$(grep -En "Removed $RELAY1_URL/ from the relay map" "$SERVER_LOG" | head -1 | cut -d: -f1)" + restored_line="$(grep -En "Home relay connection restored on $PROXY_URL/" "$SERVER_LOG" | head -1 | cut -d: -f1)" + if (( restored_line < removed_line )); then + note "server re-homed before the failover acted (lines $restored_line < $removed_line)" + rc=1 + fi +fi +if [[ "$rc" -eq 0 ]] && ! kill -0 "$SERVER_PID" 2>/dev/null; then + note "server process died" + rc=1 +fi +# The held connection rode it out: still alive, still echoing after the +# relay-map change, every echo since then over the direct path, none failed. +if [[ "$rc" -eq 0 ]]; then + wait_for_held_echoes_past $(( ECHOES_AT_REMOVAL + 1 )) $(( HELD_ECHO_INTERVAL * 4 )) || { + note "the held client stopped echoing after the failover (rc=$?)"; rc=1; } +fi +if [[ "$rc" -eq 0 ]] && grep -Eq "echo #[0-9]+ failed|Connection failed" "$HELD_LOG"; then + note "a held echo failed during the outage" + rc=1 +fi +if [[ "$rc" -eq 0 ]] && grep "Echo #[0-9]* OK via" "$HELD_LOG" | tail -n +"$(( ECHOES_AT_REMOVAL + 1 ))" | grep -Evq "via Direct "; then + note "a held echo after the failover did not run over the direct path" + rc=1 +fi +# What the held connection's paths look like after the re-home: the direct +# path selected, the wedged relay1 path gone. iroh (1.1) does not add the +# peer's NEW home relay to an established connection: a connection's relay +# paths come from the addresses the dialer knew and from address lookup, and +# a home-relay change is not signalled on the connection itself. So the +# re-home is confirmed on the server's own status above and, from the +# outside, by the fresh client below connecting via relay2, not by a relay2 +# path appearing here. +if [[ "$rc" -eq 0 ]]; then + held_paths="$(grep -E "Echo #[0-9]+ OK via" "$HELD_LOG" | tail -1 | grep -Eo "\[paths: .*\]")" + note "held connection paths after the failover: $held_paths" + if ! grep -Eq "Direct [^ ]+ \(selected" <<<"$held_paths"; then + note "the held connection does not have its direct path selected" + rc=1 + fi + if grep -q "Relay $RELAY1_URL/" <<<"$held_paths"; then + note "the held connection still holds a path through the wedged relay1" + rc=1 + fi +fi +if [[ "$rc" -eq 0 ]]; then + note "held client echoed $(held_echo_count) times, $(( $(held_echo_count) - ECHOES_AT_REMOVAL )) of them after the relay map changed" +fi +# Off-LAN reachability is back: a new relay-only client reaches the server +# through relay2 (dead port in the second slot, as in C1). +if [[ "$rc" -eq 0 ]]; then + connect_and_echo_via "$PROXY_URL" 3 "$PROXY_URL" "$DEAD_URL" || rc=1 +fi +[[ "$rc" -eq 0 ]] || { dump_log "$SERVER_LOG"; dump_log "$HELD_LOG"; } +record D1 "$rc" + +scenario D2 "the held client disconnects and a client with both relays (relay1 still the fake) redials: it comes online, reaches the server via relay2, and goes direct again" +rc=0 +stop_held_client +# A client with the product's configuration, restarted during the outage: +# relay1 (the fake) and relay2 (proxied), direct paths allowed. It has no +# failover of its own to lean on, so this is what a LAN client sees. +start_held_client "$RELAY1_URL" "$PROXY_URL" +wait_for_log_or_death "$HELD_PID" "$HELD_LOG" "Echo OK" "$READY_TIMEOUT" || { + note "the redialing client did not connect (rc=$?)"; rc=1; } +# What makes it possible: the startup probe found relay1 not connectable and +# the endpoint was bound without it, so it homed on relay2 and came online. +if [[ "$rc" -eq 0 ]] && ! grep -Eq "Binding without 1 of 2 custom relays: $RELAY1_URL/" "$HELD_LOG"; then + note "the redialing client did not leave relay1 out of its relay map" + rc=1 +fi +if [[ "$rc" -eq 0 ]] && ! grep -Eq "Connected to [0-9a-f]+ via Relay $PROXY_URL/" "$HELD_LOG"; then + note "the redialing client did not connect through relay2" + rc=1 +fi +if [[ "$rc" -eq 0 ]]; then + wait_for_log_or_death "$HELD_PID" "$HELD_LOG" "Echo #[0-9]+ OK via Direct " 30 || { + note "the redialing client's echoes never moved onto a direct path"; rc=1; } +fi +if [[ "$rc" -eq 0 ]]; then + note "redialing client connected $(grep -Eo 'via Relay .*' "$HELD_LOG" | head -1)" + note "then $(grep -E 'Echo #[0-9]+ OK via Direct ' "$HELD_LOG" | head -1 | grep -Eo 'via .*')" +fi +[[ "$rc" -eq 0 ]] || { dump_log "$HELD_LOG"; dump_log "$SERVER_LOG"; } +record D2 "$rc" + +stop_held_client +stop_server +stop_fake_relay +stop_delay_proxy +stop_relay 2 + # --------------------------------------------------------------------------- # Summary # --------------------------------------------------------------------------- diff --git a/examples/e2e/main.rs b/examples/e2e/main.rs index 95c6b89..c19da34 100644 --- a/examples/e2e/main.rs +++ b/examples/e2e/main.rs @@ -3,14 +3,17 @@ //! //! It is deliberately not an application. A `server` binds an endpoint with //! the shared builder, runs the shared home-relay failover beside its accept -//! loop, and answers one request per connection: the endpoint-bound auth -//! transcript from [`flexaccess_iroh::auth`] followed by an echo of the -//! client's message. A `client` builds an ephemeral endpoint the same way, -//! dials the server through the configured relays, proves its key, and exits -//! `0` on a clean echo or [`EXIT_AUTH_REJECTED`] when the server refuses the -//! key. Everything an application would add on top — a product ALPN, QUIC -//! tuning, config files, forwarding — is left out so a failure here is a -//! failure of this crate or of iroh, never of a product. +//! loop, and answers each connection with the endpoint-bound auth transcript +//! from [`flexaccess_iroh::auth`] followed by an echo of the client's message, +//! then echoes every further stream the client opens. A `client` builds an +//! ephemeral endpoint the same way, dials the server through the configured +//! relays, proves its key, and exits `0` on a clean echo or +//! [`EXIT_AUTH_REJECTED`] when the server refuses the key; with `--hold-secs` +//! it keeps the authenticated connection open and echoes again every few +//! seconds, logging the path each echo ran over, so a script can watch a live +//! connection through a relay outage. Everything an application would add on +//! top — a product ALPN, QUIC tuning, config files, forwarding — is left out +//! so a failure here is a failure of this crate or of iroh, never of a product. //! //! The remaining subcommands are the test fixtures the relay-failover suite //! needs, kept in Rust so the suite depends on nothing but `cargo` and @@ -22,7 +25,7 @@ use anyhow::{Context, Result, bail}; use clap::{Args, Parser, Subcommand}; use flexaccess_iroh::auth::{ClientKey, verify_endpoint_id_signature}; -use flexaccess_iroh::endpoint::{EndpointOptions, create_endpoint, endpoint_builder}; +use flexaccess_iroh::endpoint::{CreatedEndpoint, EndpointOptions, create_endpoint, endpoint_builder}; use flexaccess_iroh::flexaccess_keys::{self, AuthorizedKeys, PrivateKey, PublicKey}; use flexaccess_iroh::relay::RelayConfig; use flexaccess_iroh::relay_failover::fail_over_home_relay; @@ -116,6 +119,18 @@ impl RelayArgs { if self.relay_only && !config.is_custom() { bail!("--relay-only requires custom relays (--relay-url, at least two)"); } + // Everything here runs on one host. With mDNS compiled in, a client + // with direct paths allowed would find the server over mDNS and dial + // it directly, so the relays (and the failover behind a direct + // connection) would never be exercised. Relay-only mode adds no + // address lookup at all and is unaffected. + if !self.relay_only && cfg!(feature = "mdns") { + bail!( + "the harness was built with the `mdns` feature; with direct paths allowed, \ + mDNS would bypass the relays entirely. Build it without features \ + (cargo build --example e2e)" + ); + } Ok(config) } @@ -157,6 +172,14 @@ struct ClientArgs { /// Seconds to wait for the connection to the server. #[arg(long, default_value_t = 30)] connect_timeout: u64, + /// After the first echo, keep the connection open for this many seconds, + /// echoing again every `--echo-interval-secs` on a fresh stream and + /// logging the path each echo ran over. Any failed echo exits non-zero. + #[arg(long)] + hold_secs: Option, + /// Seconds between echoes while holding the connection open. + #[arg(long, default_value_t = 5)] + echo_interval_secs: u64, } #[derive(Args)] @@ -226,13 +249,16 @@ async fn run_server(args: ServerArgs) -> Result<()> { let builder = endpoint_builder(&relay_config, args.relay.options(true)) .alpns(vec![ALPN.to_vec()]) .secret_key(secret); - let endpoint = create_endpoint(&relay_config, builder).await?; + let CreatedEndpoint { + endpoint, + relays_left_out, + } = create_endpoint(&relay_config, builder).await?; info!("Waiting for clients to connect"); let authorized = Arc::new(authorized); let outcome = tokio::select! { outcome = accept_loop(&endpoint, &authorized) => outcome, - () = fail_over_home_relay(&endpoint, &relay_config) => Ok(()), + () = fail_over_home_relay(&endpoint, &relay_config, &relays_left_out) => Ok(()), }; endpoint.close().await; outcome @@ -260,7 +286,8 @@ async fn accept_loop(endpoint: &Endpoint, authorized: &Arc) -> R bail!("the endpoint stopped accepting connections") } -/// One request per connection: the auth line, then the message to echo. +/// The first stream carries the auth line and the message to echo; every +/// later stream on the same authenticated connection is echoed as is. async fn serve(incoming: Incoming, authorized: &AuthorizedKeys) -> Result<()> { let conn: Connection = incoming.await.context("accepting connection")?; let remote = conn.remote_id(); @@ -274,25 +301,47 @@ async fn serve(incoming: Incoming, authorized: &AuthorizedKeys) -> Result<()> { let auth_line = lines.next().context("request without an auth line")?; let message = lines.next().unwrap_or_default(); - let response = match authenticate(auth_line, &remote, authorized) { - Ok(comment) => { - info!("Client {remote} authenticated successfully as {comment}"); - format!("OK {comment}\n{message}\n") - } + let comment = match authenticate(auth_line, &remote, authorized) { + Ok(comment) => comment, Err(reason) => { // The client learns only that its proof failed, never which check. warn!("Rejected client {remote}: {reason}"); - "REJECTED Invalid authentication proof\n".to_string() + send.write_all(b"REJECTED Invalid authentication proof\n") + .await + .context("writing response")?; + send.finish().context("finishing response")?; + // The client closes once it has read the response; give it a + // moment so the response is not lost to an early close from + // this side. + let _ = tokio::time::timeout(CLIENT_CLOSE_GRACE, conn.closed()).await; + return Ok(()); } }; - send.write_all(response.as_bytes()) + info!("Client {remote} authenticated successfully as {comment}"); + send.write_all(format!("OK {comment}\n{message}\n").as_bytes()) .await .context("writing response")?; send.finish().context("finishing response")?; - // The client closes once it has read the response; give it a moment so - // the response is not lost to an early close from this side. - let _ = tokio::time::timeout(CLIENT_CLOSE_GRACE, conn.closed()).await; - Ok(()) + + // Further echoes until the client closes; a one-shot client closes right + // after reading its response, a holding client keeps opening streams. + let mut echoes = 0u64; + loop { + let (mut send, mut recv) = match conn.accept_bi().await { + Ok(stream) => stream, + Err(e) => { + info!("Client {remote} disconnected after {echoes} further echo(es) ({e})"); + return Ok(()); + } + }; + let payload = recv + .read_to_end(MAX_MESSAGE) + .await + .context("reading echo request")?; + send.write_all(&payload).await.context("writing echo")?; + send.finish().context("finishing echo")?; + echoes += 1; + } } /// Check ` ` against the connection's @@ -340,7 +389,9 @@ async fn run_client(args: ClientArgs) -> Result { .into(); let relay_config = args.relay.resolve()?; let builder = endpoint_builder(&relay_config, args.relay.options(false)); - let endpoint = create_endpoint(&relay_config, builder).await?; + // A client runs no failover: a relay left out at startup stays out for + // this (short-lived) process. + let CreatedEndpoint { endpoint, .. } = create_endpoint(&relay_config, builder).await?; let outcome = exchange(&endpoint, &args, &relay_config, &key).await; endpoint.close().await; outcome @@ -384,13 +435,13 @@ async fn exchange( .read_to_end(MAX_MESSAGE) .await .context("reading response")?; - conn.close(0u32.into(), b"done"); let response = String::from_utf8(response).context("response is not UTF-8")?; let mut lines = response.lines(); let status = lines.next().context("empty response")?; if let Some(reason) = status.strip_prefix("REJECTED ") { error!("Authentication rejected: {reason}"); + conn.close(0u32.into(), b"done"); return Ok(ExitCode::from(EXIT_AUTH_REJECTED)); } let comment = status @@ -402,26 +453,98 @@ async fn exchange( bail!("echo mismatch: sent {:?}, got {echoed:?}", args.message); } info!("Echo OK ({} bytes)", args.message.len()); + if let Some(hold) = args.hold_secs { + hold_open(&conn, args, Duration::from_secs(hold)).await?; + } + conn.close(0u32.into(), b"done"); Ok(ExitCode::SUCCESS) } -/// The selected paths of a connection, e.g. `Relay http://127.0.0.1:3340/` -/// or `Direct 127.0.0.1:41234`. +/// Keep the authenticated connection open for `hold`, echoing the message +/// on a fresh stream every `--echo-interval-secs` and logging the path each +/// echo ran over (`Echo #n OK via Direct …`). A script watches these lines +/// to see a live connection ride out a relay outage. The first failed echo +/// is an error. +async fn hold_open(conn: &Connection, args: &ClientArgs, hold: Duration) -> Result<()> { + let interval = Duration::from_secs(args.echo_interval_secs); + let deadline = tokio::time::Instant::now() + hold; + info!( + "Holding the connection open for {}s, echoing every {}s", + hold.as_secs(), + interval.as_secs() + ); + let mut echoes = 0u64; + while tokio::time::Instant::now() < deadline { + tokio::time::sleep(interval).await; + echoes += 1; + echo_once(conn, &args.message) + .await + .with_context(|| format!("echo #{echoes} failed (paths: {})", describe_paths(conn)))?; + info!("Echo #{echoes} OK via {}", describe_paths(conn)); + } + info!("Held the connection open for {}s ({echoes} further echoes)", hold.as_secs()); + Ok(()) +} + +/// One echo on a fresh stream of an authenticated connection. +async fn echo_once(conn: &Connection, message: &str) -> Result<()> { + let (mut send, mut recv) = conn.open_bi().await.context("opening stream")?; + send.write_all(message.as_bytes()) + .await + .context("sending message")?; + send.finish().context("finishing message")?; + let echoed = recv + .read_to_end(MAX_MESSAGE) + .await + .context("reading echo")?; + if echoed != message.as_bytes() { + bail!( + "echo mismatch: sent {message:?}, got {:?}", + String::from_utf8_lossy(&echoed) + ); + } + Ok(()) +} + +/// The paths of a connection: the selected one(s) first, as `via Relay +/// http://127.0.0.1:3340/` or `via Direct 127.0.0.1:41234`, then every path +/// iroh holds for it (selected or not) with its RTT, e.g. `[paths: Direct +/// 127.0.0.1:41234 (selected, rtt 1ms), Relay http://127.0.0.1:3340/ (rtt +/// 2ms)]`. A relay path names the peer's home relay as this side knows it, +/// so the full list shows a re-home even while a direct path carries the +/// traffic. fn describe_paths(conn: &Connection) -> String { let paths = conn.paths(); let selected: Vec = paths .iter() .filter(|path| path.is_selected()) - .map(|path| match path.remote_addr() { - TransportAddr::Relay(url) => format!("Relay {url}"), - TransportAddr::Ip(addr) => format!("Direct {addr}"), - other => format!("{other:?}"), + .map(|path| describe_addr(path.remote_addr())) + .collect(); + let all: Vec = paths + .iter() + .map(|path| { + let rtt = path.rtt(); + let addr = describe_addr(path.remote_addr()); + if path.is_selected() { + format!("{addr} (selected, rtt {rtt:.0?})") + } else { + format!("{addr} (rtt {rtt:.0?})") + } }) .collect(); - if selected.is_empty() { + let selected = if selected.is_empty() { "no selected path yet".to_string() } else { selected.join(", ") + }; + format!("{selected} [paths: {}]", all.join(", ")) +} + +fn describe_addr(addr: &TransportAddr) -> String { + match addr { + TransportAddr::Relay(url) => format!("Relay {url}"), + TransportAddr::Ip(addr) => format!("Direct {addr}"), + other => format!("{other:?}"), } } diff --git a/src/endpoint.rs b/src/endpoint.rs index 9abcacc..1db0455 100644 --- a/src/endpoint.rs +++ b/src/endpoint.rs @@ -12,11 +12,11 @@ use crate::relay::{RELAY_CONNECT_TIMEOUT, RelayConfig, probe_custom_relays}; use anyhow::{Context, Result}; use iroh::{ - Endpoint, + Endpoint, RelayMode, RelayUrl, address_lookup::{DnsAddressLookup, PkarrPublisher}, endpoint::{Builder as EndpointBuilder, QuicTransportConfig, presets}, }; -use log::info; +use log::{info, warn}; use std::sync::Arc; /// What an application decides about every endpoint it builds. @@ -108,18 +108,66 @@ async fn wait_online(endpoint: &Endpoint) -> Result<()> { } } +/// What [`create_endpoint`] hands back: the bound, online endpoint and the +/// custom relays it was bound without. +#[derive(Debug)] +pub struct CreatedEndpoint { + pub endpoint: Endpoint, + /// The configured custom relays that failed the startup probe and were + /// left out of the endpoint's relay map (see [`create_endpoint`]). Empty + /// with the default relays or when every custom relay probed fine. Hand + /// them to [`crate::relay_failover::fail_over_home_relay`], which puts + /// each one back into the relay map once it is connectable again; a + /// process that does not run the failover keeps them out for its + /// lifetime. + pub relays_left_out: Vec, +} + /// Create an endpoint: log the relay setup, probe every custom relay (fail -/// only if none is reachable; see [`probe_custom_relays`]), bind, and require -/// the endpoint to come online. On failure after binding the endpoint is -/// closed before the error propagates (dropping a bound endpoint without -/// `close()` is fatal under `panic = "abort"`). -pub async fn create_endpoint(relay_config: &RelayConfig, builder: EndpointBuilder) -> Result { +/// only if none is reachable; see [`probe_custom_relays`]), bind **without** +/// the relays that failed the probe, and require the endpoint to come online. +/// +/// Leaving a failed relay out of the relay map is what lets the endpoint come +/// online during that relay's outage: iroh picks its home relay by probe +/// latency, so a relay that still answers probes but cannot be connected (the +/// outage [`crate::relay_failover`] exists for) would otherwise be preferred, +/// never connect, and keep [`Endpoint::online`] pending until the timeout +/// here fails the whole start. The relay comes back through the failover's +/// restore probe; see [`CreatedEndpoint::relays_left_out`]. +/// +/// On failure after binding the endpoint is closed before the error +/// propagates (dropping a bound endpoint without `close()` is fatal under +/// `panic = "abort"`). +pub async fn create_endpoint( + relay_config: &RelayConfig, + builder: EndpointBuilder, +) -> Result { relay_config.log_status(); - probe_custom_relays(relay_config).await?; + let relays_left_out = probe_custom_relays(relay_config).await?; + let builder = if relays_left_out.is_empty() { + builder + } else { + warn!( + "Binding without {} of {} custom relays: {}", + relays_left_out.len(), + relay_config.custom_urls().len(), + relays_left_out + .iter() + .map(ToString::to_string) + .collect::>() + .join(", ") + ); + builder.relay_mode(RelayMode::Custom( + relay_config.relay_map_without(&relays_left_out), + )) + }; let endpoint = builder.bind().await.context("Failed to create iroh endpoint")?; if let Err(e) = wait_online(&endpoint).await { endpoint.close().await; return Err(e); } - Ok(endpoint) + Ok(CreatedEndpoint { + endpoint, + relays_left_out, + }) } diff --git a/src/relay.rs b/src/relay.rs index b884800..96974e2 100644 --- a/src/relay.rs +++ b/src/relay.rs @@ -170,16 +170,25 @@ impl RelayConfig { match self { Self::Default => RelayMode::Default, Self::Custom { urls, auth_token } => { - let map = RelayMap::from_iter(urls.iter().cloned()); - let map = match auth_token { - Some(token) => map.with_auth_token(token.clone()), - None => map, - }; - RelayMode::Custom(map) + RelayMode::Custom(relay_map(urls.iter().cloned(), auth_token.as_deref())) } } } + /// The custom relay map without `excluded`: what an endpoint is bound + /// with when some configured relays failed the startup probe (see + /// [`crate::endpoint::create_endpoint`]). The auth token is applied as in + /// [`Self::relay_mode`]. Empty for the default relays. + pub fn relay_map_without(&self, excluded: &[RelayUrl]) -> RelayMap { + match self { + Self::Default => RelayMap::empty(), + Self::Custom { urls, auth_token } => relay_map( + urls.iter().filter(|url| !excluded.contains(url)).cloned(), + auth_token.as_deref(), + ), + } + } + /// Log which relays are in use (silent for the default relays). Only ever /// reports *whether* an auth token is set — never the token itself. pub fn log_status(&self) { @@ -195,6 +204,15 @@ impl RelayConfig { } } +/// A relay map of `urls`, with `auth_token` (when set) applied to every relay. +fn relay_map(urls: impl IntoIterator, auth_token: Option<&str>) -> RelayMap { + let map = RelayMap::from_iter(urls); + match auth_token { + Some(token) => map.with_auth_token(token.to_string()), + None => map, + } +} + /// Build a minimal, relay-only endpoint for probing a single relay. /// /// It uses an ephemeral identity (no persistent secret, no address publishing) @@ -208,11 +226,7 @@ fn probe_endpoint_builder( relay_url: &RelayUrl, auth_token: Option<&str>, ) -> iroh::endpoint::Builder { - let map = RelayMap::from_iter([relay_url.clone()]); - let map = match auth_token { - Some(token) => map.with_auth_token(token.to_string()), - None => map, - }; + let map = relay_map([relay_url.clone()], auth_token); // iroh 1.x requires the crypto provider to be set explicitly on the // builder when starting from the `Empty` preset — the `tls-ring` feature // only makes the ring backend available, it does not wire it in. @@ -245,21 +259,27 @@ pub(crate) async fn probe_relay(relay_url: &RelayUrl, auth_token: Option<&str>) /// Probe every configured custom relay individually (in parallel). Startup /// fails only if **every** relay is unreachable; each relay that does not -/// come online is reported as a warning, since the endpoint starts with that -/// much less failover headroom. +/// come online is reported as a warning and returned, so the caller can bind +/// the endpoint without it. Default relays are not probed (returns an empty +/// list immediately). /// -/// Probing each relay on its own is what makes the warnings possible: a -/// single endpoint-wide `online()` wait proves only that *one* relay (the -/// home relay) connected and says nothing about the others. Default relays -/// are not probed (returns `Ok(())` immediately). +/// Probing each relay on its own is what makes this possible: a single +/// endpoint-wide `online()` wait proves only that *one* relay (the home +/// relay) connected and says nothing about the others. /// /// A relay that is down at startup must not stop the process: with at least -/// [`MIN_CUSTOM_RELAYS`] distinct relays configured, the remaining ones carry it, and -/// refusing to start would turn a survivable relay outage into an outage of -/// every client that restarts during it. -pub async fn probe_custom_relays(relay_config: &RelayConfig) -> Result<()> { +/// [`MIN_CUSTOM_RELAYS`] distinct relays configured, the remaining ones carry +/// it, and refusing to start would turn a survivable relay outage into an +/// outage of every client that restarts during it. Nor may it stay in the +/// relay map: iroh picks its home relay by probe latency, and a relay that +/// answers probes but cannot be connected (the outage shape of +/// [`crate::relay_failover`]) would be preferred, never connect, and keep the +/// endpoint from ever coming online — every client that restarts during such +/// an outage would then fail to start, even though the other relay works and +/// the server has already failed over to it. +pub async fn probe_custom_relays(relay_config: &RelayConfig) -> Result> { let RelayConfig::Custom { urls, auth_token } = relay_config else { - return Ok(()); + return Ok(Vec::new()); }; let token = auth_token.as_deref(); info!("Probing {} custom relays for reachability...", urls.len()); @@ -268,27 +288,36 @@ pub async fn probe_custom_relays(relay_config: &RelayConfig) -> Result<()> { .map(|url| async move { (url, probe_relay(url, token).await) }), ) .await; - let failures: Vec = results + let failures: Vec<(RelayUrl, anyhow::Error)> = results .into_iter() - .filter_map(|(url, res)| res.err().map(|e| format!("{url}: {e}"))) + .filter_map(|(url, res)| res.err().map(|e| (url.clone(), e))) .collect(); + let describe = |failures: &[(RelayUrl, anyhow::Error)]| { + failures + .iter() + .map(|(url, e)| format!("{url}: {e}")) + .collect::>() + .join("\n ") + }; if failures.len() == urls.len() { anyhow::bail!( "all {} custom relays failed to come online:\n {}", urls.len(), - failures.join("\n ") + describe(&failures) ); } if !failures.is_empty() { warn!( "{} of {} custom relays failed to come online; continuing with the rest, but a \ - further relay failure now has less to fail over to:\n {}", + further relay failure now has less to fail over to. They are left out of the \ + relay map so the endpoint homes on a relay that works, and are put back once \ + they are connectable again (see relay_failover):\n {}", failures.len(), urls.len(), - failures.join("\n ") + describe(&failures) ); } - Ok(()) + Ok(failures.into_iter().map(|(url, _)| url).collect()) } #[cfg(test)] @@ -302,6 +331,21 @@ mod tests { [RELAY.to_string(), RELAY2.to_string()] } + #[test] + fn relay_map_without_drops_only_the_excluded_relays_and_keeps_the_token() { + let cfg = RelayConfig::from_urls_with_token(&two(), Some("secret".to_string())).unwrap(); + let relay: RelayUrl = RELAY.parse().unwrap(); + let relay2: RelayUrl = RELAY2.parse().unwrap(); + let map = cfg.relay_map_without(std::slice::from_ref(&relay)); + assert!(!map.contains(&relay)); + assert!(map.contains(&relay2)); + assert_eq!(map.len(), 1); + assert_eq!(map.get(&relay2).unwrap().auth_token.as_deref(), Some("secret")); + let full = cfg.relay_map_without(&[]); + assert_eq!(full.len(), 2); + assert!(RelayConfig::Default.relay_map_without(&[relay]).is_empty()); + } + #[test] fn empty_urls_no_token_is_default() { let cfg = RelayConfig::from_urls_with_token(&[], None).unwrap(); diff --git a/src/relay_failover.rs b/src/relay_failover.rs index 3f8e129..5ba7223 100644 --- a/src/relay_failover.rs +++ b/src/relay_failover.rs @@ -41,6 +41,11 @@ //! one), the endpoint is nudged with a no-op relay-map change instead, which //! forces a fresh report the same way. //! +//! Relays that failed the startup probe were never in the map +//! ([`crate::endpoint::create_endpoint`] binds without them, so that a process +//! starting during an outage comes online on a relay that works). They are +//! handed in as already removed and get the same restore probe. +//! //! Only the *home* relay matters: non-home relays are connected on demand and //! dropped after a minute idle, which is normal and not an outage. With the //! default relays the future never resolves and never acts: there @@ -72,10 +77,17 @@ const RELAY_OUTAGE_LOG_GRACE: Duration = Duration::from_secs(5); pub const RELAY_RESTORE_INTERVAL: Duration = Duration::from_secs(90); /// Watch `endpoint`'s home relay and fail over in place when it is lost for -/// [`RELAY_OUTAGE_FAILOVER`]; see the module docs. Never resolves: run it +/// [`RELAY_OUTAGE_FAILOVER`]; see the module docs. `relays_left_out` are the +/// configured relays the endpoint was bound without +/// ([`crate::endpoint::CreatedEndpoint::relays_left_out`]); they are probed +/// for restoration like a relay removed here. Never resolves: run it /// alongside the accept loop and drop it with the endpoint. Pending forever /// with the default relays. -pub async fn fail_over_home_relay(endpoint: &Endpoint, relay_config: &RelayConfig) { +pub async fn fail_over_home_relay( + endpoint: &Endpoint, + relay_config: &RelayConfig, + relays_left_out: &[RelayUrl], +) { let RelayConfig::Custom { urls, auth_token } = relay_config else { std::future::pending().await }; @@ -89,10 +101,31 @@ pub async fn fail_over_home_relay(endpoint: &Endpoint, relay_config: &RelayConfi urls: urls.clone(), auth_token: auth_token.clone(), }; + let out: Vec = relays_left_out + .iter() + .filter(|url| { + let configured = urls.contains(url); + if !configured { + log::error!("{url} was left out at startup but is not a configured relay; ignoring it"); + } + configured + }) + .cloned() + .collect(); + if !out.is_empty() { + log::info!( + "{} relay(s) left out of the relay map at startup will be probed for restoration \ + every {}s: {}", + out.len(), + RELAY_RESTORE_INTERVAL.as_secs(), + out.iter().map(ToString::to_string).collect::>().join(", ") + ); + } run_failover( endpoint.home_relay_status(), |statuses| describe_statuses(statuses), &mut relays, + out, ) .await } @@ -330,17 +363,21 @@ async fn fail_over( } /// The failover loop proper, generic over the status source and the relay -/// map so tests can drive it with doubles. -async fn run_failover(mut watcher: W, describe: D, relays: &mut R) +/// map so tests can drive it with doubles. `out_at_start` are relays already +/// out of the map when the loop starts. +async fn run_failover(mut watcher: W, describe: D, relays: &mut R, out_at_start: Vec) where W: Watcher, D: Fn(&W::Value) -> HomeRelay, R: FailoverRelays, { let mut outage = Outage::default(); - // Relays taken out of the map, each with when it is next probed for + // Relays out of the map, each with when it is next probed for // restoration. - let mut removed: Vec<(RelayUrl, Instant)> = Vec::new(); + let mut removed: Vec<(RelayUrl, Instant)> = out_at_start + .into_iter() + .map(|url| (url, Instant::now() + RELAY_RESTORE_INTERVAL)) + .collect(); let mut value = watcher.get(); outage.observe(&describe(&value)); loop { @@ -523,7 +560,21 @@ mod tests { /// Run the loop against `status` for `bound` of paused time. async fn run_for(status: &Watchable, relays: &mut FakeRelays, bound: Duration) { - let _ = tokio::time::timeout(bound, run_failover(status.watch(), describe, relays)).await; + run_for_with(status, relays, bound, Vec::new()).await; + } + + /// `run_for` with relays already out of the map at the start. + async fn run_for_with( + status: &Watchable, + relays: &mut FakeRelays, + bound: Duration, + out_at_start: Vec, + ) { + let _ = tokio::time::timeout( + bound, + run_failover(status.watch(), describe, relays, out_at_start), + ) + .await; } const SEC: Duration = Duration::from_secs(1); @@ -695,6 +746,70 @@ mod tests { ); } + #[tokio::test(start_paused = true)] + async fn a_relay_left_out_at_startup_is_restored_once_connectable_and_never_removed_again() { + // Healthy on B from the start, A left out at startup: A gets the + // restore probe (still down at 90 s, back at 180 s) and nothing is + // ever removed or nudged. + let status = Watchable::new(Status::UpOnB); + let mut relays = FakeRelays::new(&status, None); + relays.connectable.store(false, Ordering::SeqCst); + let connectable = relays.connectable.clone(); + let flipper = async move { + tokio::time::sleep(RELAY_RESTORE_INTERVAL + 5 * SEC).await; + connectable.store(true, Ordering::SeqCst); + }; + tokio::join!( + run_for_with( + &status, + &mut relays, + RELAY_RESTORE_INTERVAL * 3, + vec![relay_a()] + ), + flipper + ); + assert_eq!( + relays.actions(), + vec![ + Action::Restore { + url: relay_a(), + restored: false, + at: RELAY_RESTORE_INTERVAL, + }, + Action::Restore { + url: relay_a(), + restored: true, + at: RELAY_RESTORE_INTERVAL * 2, + }, + ] + ); + } + + #[tokio::test(start_paused = true)] + async fn a_relay_left_out_at_startup_is_not_removed_when_the_home_relay_wedges() { + // A left out at startup and unconnectable; the endpoint homed on B, + // which wedges: B is removed (C is still in the map), A is not + // touched beyond its restore probes. + let status = Watchable::new(Status::DownOnB); + let mut relays = FakeRelays::new(&status, None); + relays.connectable.store(false, Ordering::SeqCst); + run_for_with(&status, &mut relays, 100 * SEC, vec![relay_a()]).await; + assert_eq!( + relays.actions(), + vec![ + Action::Remove { + url: relay_b(), + at: RELAY_OUTAGE_FAILOVER, + }, + Action::Restore { + url: relay_a(), + restored: false, + at: RELAY_RESTORE_INTERVAL, + }, + ] + ); + } + #[tokio::test(start_paused = true)] async fn a_second_wedged_relay_is_removed_while_the_first_is_still_out_but_never_the_last() { // A is removed at 60 s and stays unconnectable. The endpoint moves to