diff --git a/Cargo.lock b/Cargo.lock index 1631617..7e324eb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1929,7 +1929,7 @@ dependencies = [ [[package]] name = "flextunnel-cli" -version = "0.0.69" +version = "0.0.70" dependencies = [ "anyhow", "clap", @@ -1945,7 +1945,7 @@ dependencies = [ [[package]] name = "flextunnel-core" -version = "0.0.69" +version = "0.0.70" dependencies = [ "anyhow", "askama", @@ -1961,6 +1961,7 @@ dependencies = [ "libc", "log", "n0-future", + "n0-watcher", "rand 0.9.4", "reqwest", "rustls", @@ -1976,7 +1977,7 @@ dependencies = [ [[package]] name = "flextunnel-desktop" -version = "0.0.69" +version = "0.0.70" dependencies = [ "aes-gcm", "anyhow", @@ -2005,7 +2006,7 @@ dependencies = [ [[package]] name = "flextunnel-ffi" -version = "0.0.69" +version = "0.0.70" dependencies = [ "flextunnel-core", "iroh", diff --git a/Cargo.toml b/Cargo.toml index fc61581..d616069 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ default-members = [ ] [workspace.package] -version = "0.0.69" +version = "0.0.70" edition = "2024" description = "SOCKS5/HTTP-proxy-over-QUIC split tunnel via iroh P2P" @@ -33,6 +33,9 @@ iroh-mdns-address-lookup = "0.4.0" libc = "0.2" log = "0.4" n0-future = "0.3" +# Test double for iroh's `Watcher`-based status APIs (the relay watchdog tests +# drive a plain `Watchable`); the same crate iroh itself re-exports `Watcher` from. +n0-watcher = "1" rand = "0.9" ratatui = "0.30" # Custom-relay `/healthz` checks (see `transport::paths`). `rustls-no-provider` diff --git a/README.md b/README.md index 9373a70..c385b1c 100644 --- a/README.md +++ b/README.md @@ -619,6 +619,13 @@ Auto-reconnect is **enabled by default** (`auto_reconnect = true`); pass connecting directly; on-list requests are held for the reconnect (up to 45s) and only then fail with a network-unreachable reply. +A **server** with custom relays watches its own home-relay registration: if it +has no connected home relay for 60s it re-checks the network, and if that +has not helped by 180s it rebuilds its endpoint in place (same server id) — +the in-process equivalent of a restart, so relay-only clients (the iOS app, +anything off the LAN) are not stranded until someone restarts the service. +See [`docs/architecture.md`](docs/architecture.md#relay-watchdog-server-custom-relays). + ## Logging Logging uses `env_logger`. The default is `info` with iroh/tracing quieted to diff --git a/crates/flextunnel-cli/src/main.rs b/crates/flextunnel-cli/src/main.rs index 3e7b660..b2f9c37 100644 --- a/crates/flextunnel-cli/src/main.rs +++ b/crates/flextunnel-cli/src/main.rs @@ -13,6 +13,7 @@ use clap::{Parser, Subcommand}; use std::io::IsTerminal; use std::num::NonZeroU32; use std::path::PathBuf; +use std::pin::pin; use std::sync::Arc; use std::time::Duration; use tokio::sync::Notify; @@ -33,7 +34,9 @@ use flextunnel_core::proxy::{ }; use flextunnel_core::transport::endpoint::{ EndpointAllowlists, RelayConfig, create_server_endpoint, secret_to_endpoint_id, + server_rebuild_factory, }; +use flextunnel_core::transport::relay_watchdog; use flextunnel_core::{auth, config, secret}; #[derive(Parser)] @@ -614,6 +617,12 @@ struct QuickServer { /// relay/connection teardown must never leave the process unkillable. const SHUTDOWN_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); +/// Pause between attempts to bind a replacement endpoint after the relay +/// watchdog retired the old one and the rebuild itself failed (e.g. no route +/// to bind on). The server has no endpoint at all during this wait, so it is +/// short — there is nothing to lose by trying again soon. +const REBUILD_RETRY: Duration = Duration::from_secs(30); + /// Build the ephemeral `ServerConfig` for `server start --quick`: a full-tunnel /// routed set (`routed_domains = ["*"]`, `routed_cidrs = ["0.0.0.0/0", "::/0"]`) /// plus a freshly generated in-memory identity, returned *alongside* the config — @@ -798,9 +807,16 @@ async fn run_server( .map(|q| std::collections::HashSet::from([q.client_id])) .unwrap_or_default(), }; - let endpoint = create_server_endpoint(&relay_config, secret_key, allowlists) + let endpoint = create_server_endpoint(&relay_config, secret_key.clone(), allowlists.clone()) .await .context("Failed to create iroh endpoint")?; + // The relay watchdog's remedy of last resort: a fresh endpoint with the + // same identity and allowlists (see the serve loop below). + let rebuild = server_rebuild_factory(relay_config.clone(), secret_key, allowlists); + // Only a custom-relay server hangs its reachability on one home-relay + // registration (n0 discovery is off, clients dial by relay hint), so the + // watchdog is armed for custom relays only. + let relay_watchdog_armed = relay_config.is_custom(); log::info!("flextunnel server Node ID: {}", endpoint.id()); match &quick { @@ -844,20 +860,13 @@ async fn run_server( blocklist, first_client: first_client.clone(), }); - let run = async { - server - .run(&endpoint) - .await - .map_err(|e| anyhow::anyhow!("Server error: {e}")) - }; - // Quick-mode grace window: fire after `QUICK_IDLE_TIMEOUT` unless a client // connects first (which resolves `notified()` and parks this future // forever). `first_client` is `Some` exactly in quick mode; a normal server // parks here immediately, so the arm never fires. `notify_one` stores a // permit, so a client that connects before this future is first polled is - // not missed. - let grace = async { + // not missed. Pinned outside the serve loop so it spans endpoint rebuilds. + let mut grace = pin!(async { match &first_client { Some(notify) => { tokio::select! { @@ -867,19 +876,88 @@ async fn run_server( } None => std::future::pending::<()>().await, } - }; + }); + // One signal listener for the whole serve loop: re-registering it per pass + // could drop a signal delivered while an endpoint is being rebuilt. + let mut shutdown = pin!(app::shutdown_signal()); + + /// How one pass of the serve loop ended. + enum Pass { + /// The server is done (clean or failed): close the endpoint and return. + Exit(Result<()>), + /// The relay watchdog gave up on the endpoint after an outage this long. + Rebuild(Duration), + } + + // Serve loop. A pass serves on the current endpoint until the server ends, + // a shutdown signal arrives, the quick-mode grace expires, or — custom + // relays only — the relay watchdog reports the endpoint has lost its home + // relay for good. That last case is the in-process equivalent of the + // process restart known to fix it: close the wedged endpoint, bind a fresh + // one with the same identity, and serve again. The `ProxyServer` (its + // registries, blocklist, status state) carries over; the old endpoint's + // connections and bridge tasks end with it. + let mut endpoint = endpoint; + let res = loop { + let pass = { + let run = Arc::clone(&server).run(&endpoint); + let outage = async { + if relay_watchdog_armed { + relay_watchdog::watch_home_relay(&endpoint).await + } else { + std::future::pending().await + } + }; + tokio::select! { + res = run => Pass::Exit(res.map_err(|e| anyhow::anyhow!("Server error: {e}"))), + sig = &mut shutdown => Pass::Exit(sig.map(|()| { + log::info!("Received shutdown signal, stopping server"); + })), + _ = &mut grace => { + log::warn!("Quick mode: no client connected within 5 minutes — exiting"); + Pass::Exit(Ok(())) + } + outage = outage => Pass::Rebuild(outage), + } + }; + let outage = match pass { + Pass::Exit(res) => break res, + Pass::Rebuild(outage) => outage, + }; - let res = tokio::select! { - res = run => res, - sig = app::shutdown_signal() => { - sig?; - log::info!("Received shutdown signal, stopping server"); - Ok(()) - } - _ = grace => { - log::warn!("Quick mode: no client connected within 5 minutes — exiting"); - Ok(()) - } + log::error!( + "No connected home relay for {:.0}s despite a network re-check; rebuilding the \ + endpoint from scratch (server id stays {})", + outage.as_secs_f64(), + endpoint.id() + ); + close_endpoint_or_exit(&endpoint).await; + endpoint = loop { + match rebuild().await { + Ok(fresh) => break fresh, + Err(e) => { + log::error!( + "Endpoint rebuild failed: {e:#}; retrying in {}s", + REBUILD_RETRY.as_secs() + ); + // Nothing is bound while waiting here, so both exits + // below return directly: there is no endpoint to close. + tokio::select! { + _ = tokio::time::sleep(REBUILD_RETRY) => {} + sig = &mut shutdown => { + sig?; + log::info!("Received shutdown signal, stopping server"); + return Ok(()); + } + _ = &mut grace => { + log::warn!("Quick mode: no client connected within 5 minutes — exiting"); + return Ok(()); + } + } + } + } + }; + log::warn!("Endpoint rebuilt; serving again as {}", endpoint.id()); }; close_endpoint_or_exit(&endpoint).await; diff --git a/crates/flextunnel-core/Cargo.toml b/crates/flextunnel-core/Cargo.toml index 45f2eff..e86d84b 100644 --- a/crates/flextunnel-core/Cargo.toml +++ b/crates/flextunnel-core/Cargo.toml @@ -41,4 +41,5 @@ libc.workspace = true iroh-mdns-address-lookup.workspace = true [dev-dependencies] +n0-watcher.workspace = true tempfile.workspace = true diff --git a/crates/flextunnel-core/src/proxy/bridge.rs b/crates/flextunnel-core/src/proxy/bridge.rs index c8cb4a8..23406c1 100644 --- a/crates/flextunnel-core/src/proxy/bridge.rs +++ b/crates/flextunnel-core/src/proxy/bridge.rs @@ -106,6 +106,10 @@ impl BridgeUpstream { /// only when `endpoint` closes underneath it, failing each retry. pub async fn run(self: Arc, endpoint: Endpoint) { let name = &self.config.name; + // A previous run (on a since-closed endpoint, see the server's relay + // watchdog rebuild) may have been aborted while connected; its stale + // connection must not read as live until this run establishes its own. + *self.conn.lock().expect("bridge conn lock") = None; let mut attempt: u32 = 0; loop { if attempt > 0 { diff --git a/crates/flextunnel-core/src/proxy/server.rs b/crates/flextunnel-core/src/proxy/server.rs index bdba148..f2dc022 100644 --- a/crates/flextunnel-core/src/proxy/server.rs +++ b/crates/flextunnel-core/src/proxy/server.rs @@ -307,14 +307,22 @@ impl ProxyServer { } /// Accept connections until the endpoint closes or the server self-blocks. + /// + /// May be called again on a *fresh* endpoint after the previous one was + /// closed (the relay watchdog's rebuild): the registries and blocklist + /// carry over, the old endpoint's connection handlers end as its + /// connections close, and its bridge tasks are aborted when the previous + /// `run` future is dropped. pub async fn run(self: Arc, endpoint: &Endpoint) -> ProxyResult<()> { - // Maintain the outbound bridge upstreams for the life of the process. - // The bridging side dials out on this same server endpoint, so the TLS + // Maintain the outbound bridge upstreams for the life of this run. The + // bridging side dials out on this same server endpoint, so the TLS // identity it presents is this server's persistent id — what the target - // server's allowlist matches. The tasks retry forever and die with the - // endpoint. + // server's allowlist matches. The tasks retry forever; owning them in a + // `JoinSet` aborts them with this future, so a rebuild never leaves + // bridges retrying on a closed endpoint. + let mut bridge_tasks = tokio::task::JoinSet::new(); for bridge in &self.bridges { - tokio::spawn(bridge.clone().run(endpoint.clone())); + bridge_tasks.spawn(bridge.clone().run(endpoint.clone())); } let conn_limit = Arc::new(Semaphore::new(MAX_CONCURRENT_CONNECTIONS)); diff --git a/crates/flextunnel-core/src/transport/endpoint.rs b/crates/flextunnel-core/src/transport/endpoint.rs index b8586ad..9fa332b 100644 --- a/crates/flextunnel-core/src/transport/endpoint.rs +++ b/crates/flextunnel-core/src/transport/endpoint.rs @@ -30,7 +30,7 @@ pub const CLOSE_NOT_ALLOWLISTED: u32 = 3; /// The server's per-ALPN endpoint-id allowlists, enforced natively at the TLS /// handshake by [`AllowlistHook`]. An empty set disables its ALPN entirely — /// the allowlist is the sole and mandatory credential on both. -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub struct EndpointAllowlists { /// Servers allowed to bridge into this server over [`BRIDGE_ALPN`]. pub bridge_servers: HashSet, @@ -436,15 +436,7 @@ pub async fn create_server_endpoint( // no-op for the default relays. probe_custom_relays(relay_config).await?; - let builder = create_endpoint_builder(relay_config, Some(&secret))? - .alpns(vec![ALPN.to_vec(), BRIDGE_ALPN.to_vec(), QUICK_ALPN.to_vec()]) - .hooks(AllowlistHook::new(allowlists)) - .secret_key(secret); - - let endpoint = builder - .bind() - .await - .context("Failed to create iroh endpoint")?; + let endpoint = bind_server_endpoint(relay_config, secret, allowlists).await?; if let Err(e) = wait_online(&endpoint).await { // Close before propagating: dropping a bound endpoint without @@ -455,6 +447,52 @@ pub async fn create_server_endpoint( Ok(endpoint) } +/// Bind a server endpoint: persistent identity, all three server ALPNs, the +/// allowlist hook. No relay probe, no online wait — [`create_server_endpoint`] +/// and [`server_rebuild_factory`] layer their own policy over this. +async fn bind_server_endpoint( + relay_config: &RelayConfig, + secret: SecretKey, + allowlists: EndpointAllowlists, +) -> Result { + let builder = create_endpoint_builder(relay_config, Some(&secret))? + .alpns(vec![ALPN.to_vec(), BRIDGE_ALPN.to_vec(), QUICK_ALPN.to_vec()]) + .hooks(AllowlistHook::new(allowlists)) + .secret_key(secret); + builder.bind().await.context("Failed to create iroh endpoint") +} + +/// The rebuild recipe for the server endpoint, used when the relay watchdog +/// (`transport::relay_watchdog`) gives up on the current one. Same identity +/// and allowlists as the original, so the server's id — what clients dial — +/// never changes. Differs from first creation the same way the client's +/// rebuild does: +/// +/// - **No per-relay probe.** Creation fails fast if *any* relay is down +/// (configuration validation); mid-outage that strictness would block +/// recovery through the one relay that still answers. +/// - **The online wait is tolerated failing.** A fresh endpoint is no worse +/// than the wedged one it replaces — LAN clients can still find it over +/// mDNS — and the watchdog trips again if the relays stay unreachable. +pub fn server_rebuild_factory( + relay_config: RelayConfig, + secret: SecretKey, + allowlists: EndpointAllowlists, +) -> EndpointFactory { + Arc::new(move || { + let relay_config = relay_config.clone(); + let secret = secret.clone(); + let allowlists = allowlists.clone(); + Box::pin(async move { + let endpoint = bind_server_endpoint(&relay_config, secret, allowlists).await?; + if let Err(e) = wait_online(&endpoint).await { + log::warn!("Rebuilt endpoint: {e:#}; continuing (LAN discovery still works)"); + } + Ok(endpoint) + }) + }) +} + /// Bind a client endpoint: no relay probe, no online wait — callers layer /// their own creation-vs-rebuild policy over this. async fn bind_client_endpoint( diff --git a/crates/flextunnel-core/src/transport/mod.rs b/crates/flextunnel-core/src/transport/mod.rs index 4798fe5..59096b3 100644 --- a/crates/flextunnel-core/src/transport/mod.rs +++ b/crates/flextunnel-core/src/transport/mod.rs @@ -7,6 +7,7 @@ pub mod endpoint; pub mod paths; +pub mod relay_watchdog; use anyhow::{Context, Result}; use iroh::endpoint::QuicTransportConfig; diff --git a/crates/flextunnel-core/src/transport/relay_watchdog.rs b/crates/flextunnel-core/src/transport/relay_watchdog.rs new file mode 100644 index 0000000..b7a5e64 --- /dev/null +++ b/crates/flextunnel-core/src/transport/relay_watchdog.rs @@ -0,0 +1,271 @@ +//! Server-side home-relay watchdog. +//! +//! A server configured with custom relays is reachable to off-LAN clients +//! *only* through its home relay: with n0 discovery off, clients dial with +//! relay hints, and a relay forwards QUIC Initials only to endpoints currently +//! registered on it. iroh keeps that registration alive on its own, but it has +//! been observed (v1.0.3, relays behind Cloudflare tunnels that reset idle +//! WebSockets roughly hourly) to silently lose its home relay for good after +//! one such reset: no dial retries, no warnings, no registration on any relay — +//! the server just stops being dialable until the process is restarted, while +//! LAN clients that find it over mDNS keep working and mask the outage. +//! +//! [`watch_home_relay`] observes [`Endpoint::home_relay_status`] and reacts in +//! two steps, mirroring the client's reconnect escalation: +//! +//! 1. after [`RELAY_OUTAGE_NUDGE`] without a connected home relay it calls +//! [`Endpoint::network_change`], which forces a fresh net report and relay +//! re-selection (enough when only the bookkeeping went stale); +//! 2. after [`RELAY_OUTAGE_REBUILD`] it resolves, telling the caller to +//! replace the endpoint — the in-process equivalent of the restart that is +//! known to fix it. The caller (`run_server`) closes the wedged endpoint, +//! binds a fresh one with the same identity, and serves on again. +//! +//! Only the *home* relay matters: non-home relays are connected on demand and +//! dropped after a minute idle, which is normal and not an outage. + +use iroh::{Endpoint, Watcher}; +use iroh::endpoint::RelayStatus; +use std::future::Future; +use std::time::Duration; +use tokio::time::Instant; + +/// How long the endpoint may go without a connected home relay before the +/// watchdog nudges it with `network_change()`. Long enough to ride out a +/// routine relay reconnect (iroh's own reconnect backoff caps at 16s) plus the +/// ~25s cadence of its periodic net report. +pub const RELAY_OUTAGE_NUDGE: Duration = Duration::from_secs(60); + +/// How long from the start of the outage before the watchdog gives up on the +/// endpoint and asks for a rebuild. Leaves the nudge two minutes to take +/// effect (a net report through slow relays can take tens of seconds). +pub const RELAY_OUTAGE_REBUILD: Duration = Duration::from_secs(180); + +/// Watch `endpoint`'s home-relay status and resolve — with the outage's +/// duration — once it has had no connected home relay for +/// [`RELAY_OUTAGE_REBUILD`], having nudged it with `network_change()` at +/// [`RELAY_OUTAGE_NUDGE`]. Never resolves while the home relay stays +/// connected; a reconnect at any point resets the clock. Pending forever once +/// the endpoint is gone. +pub async fn watch_home_relay(endpoint: &Endpoint) -> Duration { + watch_outage( + endpoint.home_relay_status(), + |statuses| describe_statuses(statuses), + || endpoint.network_change(), + ) + .await +} + +/// Describe a home-relay status vector for the watchdog: `Ok(())` when some +/// home relay is connected, otherwise `Err(reason)` naming what is wrong. +fn describe_statuses(statuses: &[RelayStatus]) -> Result<(), String> { + if statuses.iter().any(RelayStatus::is_connected) { + return Ok(()); + } + if statuses.is_empty() { + return Err("no home relay selected".into()); + } + let parts: Vec = statuses + .iter() + .map(|s| match s.last_error() { + Some(e) => format!("{} disconnected ({e:#})", s.url()), + None => format!("{} not connected", s.url()), + }) + .collect(); + Err(parts.join("; ")) +} + +/// The watchdog proper, generic over the status source so it can be driven by +/// a plain watchable in tests. `describe` classifies a status value +/// (`Ok` = connected); `nudge` is the first-stage remedy. +async fn watch_outage(mut watcher: W, describe: D, mut nudge: N) -> Duration +where + W: Watcher, + D: Fn(&W::Value) -> Result<(), String>, + N: FnMut() -> Fut, + Fut: Future, +{ + let mut outage_since: Option = None; + let mut nudged = false; + let mut value = watcher.get(); + loop { + match describe(&value) { + Ok(()) => { + if let Some(since) = outage_since.take() { + log::info!( + "Home relay connection restored after {:.0}s", + since.elapsed().as_secs_f64() + ); + } + nudged = false; + } + Err(reason) => { + if outage_since.is_none() { + outage_since = Some(Instant::now()); + log::warn!( + "No connected home relay ({reason}); off-LAN clients cannot reach this \ + server until it reconnects" + ); + } + } + } + + let Some(since) = outage_since else { + // Healthy: nothing to time, just wait for the next status change. + value = match watcher.updated().await { + Ok(value) => value, + Err(_disconnected) => std::future::pending().await, + }; + continue; + }; + + let deadline = since + if nudged { RELAY_OUTAGE_REBUILD } else { RELAY_OUTAGE_NUDGE }; + tokio::select! { + _ = tokio::time::sleep_until(deadline) => { + if nudged { + return since.elapsed(); + } + nudged = true; + log::warn!( + "Still no connected home relay after {:.0}s; nudging the endpoint to \ + re-check its network and relays", + since.elapsed().as_secs_f64() + ); + nudge().await; + // The nudge may have already reconnected the relay; re-read + // rather than wait for a change notification we may have + // missed while it ran. + value = watcher.get(); + } + updated = watcher.updated() => { + value = match updated { + Ok(value) => value, + Err(_disconnected) => std::future::pending().await, + }; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use n0_watcher::Watchable; + use std::sync::Arc; + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Test double for the home-relay status: `true` = a home relay is + /// connected. + fn describe(connected: &bool) -> Result<(), String> { + if *connected { Ok(()) } else { Err("down".into()) } + } + + /// Run the watchdog on `status`, counting nudges. Returns the watchdog + /// future's resolution wrapped in a bounded wait so a test never hangs. + async fn run_for( + status: &Watchable, + nudges: Arc, + bound: Duration, + ) -> Option { + let watchdog = watch_outage(status.watch(), describe, || { + let nudges = nudges.clone(); + async move { + nudges.fetch_add(1, Ordering::SeqCst); + } + }); + tokio::time::timeout(bound, watchdog).await.ok() + } + + #[tokio::test(start_paused = true)] + async fn healthy_relay_never_trips() { + let status = Watchable::new(true); + let nudges = Arc::new(AtomicUsize::new(0)); + let tripped = run_for(&status, nudges.clone(), RELAY_OUTAGE_REBUILD * 3).await; + assert!(tripped.is_none(), "healthy relay must never request a rebuild"); + assert_eq!(nudges.load(Ordering::SeqCst), 0); + } + + #[tokio::test(start_paused = true)] + async fn sustained_outage_nudges_then_requests_rebuild() { + let status = Watchable::new(false); + let nudges = Arc::new(AtomicUsize::new(0)); + let elapsed = run_for(&status, nudges.clone(), RELAY_OUTAGE_REBUILD * 2) + .await + .expect("a sustained outage must request a rebuild"); + assert_eq!(nudges.load(Ordering::SeqCst), 1, "exactly one nudge before the rebuild"); + assert!(elapsed >= RELAY_OUTAGE_REBUILD); + assert!(elapsed < RELAY_OUTAGE_REBUILD + Duration::from_secs(1)); + } + + #[tokio::test(start_paused = true)] + async fn recovery_before_the_nudge_resets_the_clock() { + let status = Watchable::new(true); + let nudges = Arc::new(AtomicUsize::new(0)); + let flipper = { + let status = status.clone(); + async move { + // Drop out for half the nudge window, then recover; the + // watchdog must neither nudge nor trip. + tokio::time::sleep(Duration::from_secs(5)).await; + status.set(false).ok(); + tokio::time::sleep(RELAY_OUTAGE_NUDGE / 2).await; + status.set(true).ok(); + } + }; + let (tripped, ()) = tokio::join!( + run_for(&status, nudges.clone(), RELAY_OUTAGE_REBUILD * 2), + flipper + ); + assert!(tripped.is_none()); + assert_eq!(nudges.load(Ordering::SeqCst), 0); + } + + #[tokio::test(start_paused = true)] + async fn recovery_after_the_nudge_avoids_the_rebuild() { + let status = Watchable::new(false); + let nudges = Arc::new(AtomicUsize::new(0)); + let flipper = { + let status = status.clone(); + async move { + // Recover between the nudge and the rebuild deadline. + tokio::time::sleep(RELAY_OUTAGE_NUDGE + Duration::from_secs(10)).await; + status.set(true).ok(); + } + }; + let (tripped, ()) = tokio::join!( + run_for(&status, nudges.clone(), RELAY_OUTAGE_REBUILD * 2), + flipper + ); + assert!(tripped.is_none(), "a relay that came back must not be rebuilt"); + assert_eq!(nudges.load(Ordering::SeqCst), 1); + } + + #[tokio::test(start_paused = true)] + async fn a_second_outage_starts_a_fresh_clock() { + let status = Watchable::new(false); + let nudges = Arc::new(AtomicUsize::new(0)); + let flipper = { + let status = status.clone(); + async move { + // First outage: nudged, then recovers. Second outage: must + // get its own nudge and only trip a full window later. + tokio::time::sleep(RELAY_OUTAGE_NUDGE + Duration::from_secs(10)).await; + status.set(true).ok(); + tokio::time::sleep(Duration::from_secs(10)).await; + status.set(false).ok(); + } + }; + let start = Instant::now(); + let (tripped, ()) = tokio::join!( + run_for(&status, nudges.clone(), RELAY_OUTAGE_REBUILD * 3), + flipper + ); + let elapsed = tripped.expect("second outage must eventually trip"); + assert_eq!(nudges.load(Ordering::SeqCst), 2); + assert!(elapsed >= RELAY_OUTAGE_REBUILD); + assert!(elapsed < RELAY_OUTAGE_REBUILD + Duration::from_secs(1)); + // Second outage began at nudge + 20s; the trip comes a full window after that. + let total = start.elapsed(); + assert!(total >= RELAY_OUTAGE_NUDGE + Duration::from_secs(20) + RELAY_OUTAGE_REBUILD); + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 26b69a4..4e4fca1 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -245,6 +245,39 @@ Implemented in `ProxyClient::run` / `handle_failure`: (`TUNNEL_RECOVERY_HOLD`), deploy-style connection holding — and only then fail with a network-unreachable reply. +## Relay watchdog (server, custom relays) + +Implemented in `transport/relay_watchdog.rs`, driven by the serve loop in the +CLI's `run_server`. A custom-relay server is dialable from off the LAN only +while it is **registered on its home relay** (n0 discovery is off; clients dial +by relay hint, and a relay forwards Initials only to endpoints connected to it). +iroh (v1.0.3) has been observed to silently lose its home relay for good after +a routine relay reconnect: no dial retries, no warnings, no registration on +any relay — the server stops being reachable through the relays until the +process restarts, while LAN clients that find it over mDNS keep working and +hide the outage (the mac desktop reconnects, the iOS app times out). + +The watchdog observes `Endpoint::home_relay_status()` and escalates like the +client's reconnect loop: + +1. no connected home relay for `RELAY_OUTAGE_NUDGE` (60s) → log a warning and + call `Endpoint::network_change()` (forces a fresh net report and relay + re-selection — enough when only the bookkeeping went stale); +2. still none at `RELAY_OUTAGE_REBUILD` (180s from the outage start) → the + serve loop closes the endpoint, binds a fresh one with the **same identity + and allowlists** (`server_rebuild_factory`: no per-relay probe, online-wait + tolerated failing), and calls `ProxyServer::run` again on it. The + `ProxyServer` — registries, blocklist, status state — carries over; the old + endpoint's connections end with it, and its bridge tasks are aborted with + the previous `run` future (they are owned by a `JoinSet` per run). A failed + rebuild is retried every `REBUILD_RETRY` (30s). + +A reconnect at any point resets the outage clock. Non-home relays are +connected on demand and dropped after a minute idle, which is normal and never +counts as an outage. With the default relays the watchdog is not armed: +reachability there rests on n0 publishing/resolution, not on one relay +registration. + On every exit path both `run_server` and `run_client` call `endpoint.close().await` before the `Endpoint` drops; skipping it makes iroh tear down its relay tasks ungracefully (a `JoinSet` panic that is fatal under the @@ -301,6 +334,9 @@ defenses. | `HEARTBEAT_INTERVAL` | 10s | `transport/mod.rs` | | `LIVENESS_WINDOW` | 33s | `transport/mod.rs` | | `RELAY_CONNECT_TIMEOUT` (`endpoint.online()`) | 10s | `transport/endpoint.rs` | +| `RELAY_OUTAGE_NUDGE` (server relay watchdog) | 60s | `transport/relay_watchdog.rs` | +| `RELAY_OUTAGE_REBUILD` (server relay watchdog) | 180s | `transport/relay_watchdog.rs` | +| `REBUILD_RETRY` (server endpoint rebuild) | 30s | `flextunnel-cli/src/main.rs` | | `CONNECT_TIMEOUT` (client server connect) | 30s | `proxy/client.rs` | | `HANDSHAKE_TIMEOUT` | 10s | `proxy/client.rs`, `proxy/server.rs`, `proxy/bridge.rs` | | `LOCAL_HANDSHAKE_TIMEOUT` | 10s | `proxy/client.rs` |