diff --git a/Cargo.lock b/Cargo.lock index 6d3ab40..ec0b47f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -922,7 +922,7 @@ dependencies = [ [[package]] name = "ezvpn" -version = "0.0.46" +version = "0.0.47" dependencies = [ "android_logger", "anyhow", @@ -997,8 +997,8 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flexaccess-iroh" -version = "0.0.3" -source = "git+https://github.com/flexaccessdev/flexaccess-iroh?tag=v0.0.3#422dacee5b38a667bd6da2b709d28d86c8d7661f" +version = "0.0.8" +source = "git+https://github.com/flexaccessdev/flexaccess-iroh?tag=v0.0.8#e718f5d0b00f9837a7bea50114d77320f31f5197" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index f45baa7..31bfa99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ezvpn" -version = "0.0.46" +version = "0.0.47" edition = "2024" description = "IP-over-QUIC VPN tunnel via iroh P2P" readme = "README.md" @@ -34,7 +34,7 @@ ipnet = { version = "2", features = ["serde"] } # Shared iroh transport layer (RelayConfig + per-relay probe, endpoint # build/rebuild, home-relay watchdog, endpoint-bound auth transcript). No # `mdns` feature: ezvpn runs no local-network discovery. -flexaccess-iroh = { git = "https://github.com/flexaccessdev/flexaccess-iroh", tag = "v0.0.3" } +flexaccess-iroh = { git = "https://github.com/flexaccessdev/flexaccess-iroh", tag = "v0.0.8" } # Redirected to the fork below via `[patch.crates-io]`, which also covers the # shared crate's own `iroh` dependency so the graph holds a single `iroh`. iroh = "1.1.0" diff --git a/README.md b/README.md index f4de76e..9d8ea2f 100644 --- a/README.md +++ b/README.md @@ -722,12 +722,12 @@ the same IP during reconnects. Reassignment is expected mainly after server restart or allocation state changes. A **server** with custom relays watches its own home-relay registration: if it -has no connected home relay for 60s it re-checks the network, and if that has -not helped by 180s it rebuilds its endpoint in place (same node id) — the -in-process equivalent of a restart, so relay-only clients (the mobile apps, -anything off the LAN) are not stranded with connect timeouts until someone -restarts the service. See -[`docs/Architecture.md`](docs/Architecture.md#relay-watchdog-server-custom-relays). +has no connected home relay for 60s and iroh has not re-homed it on its own, +it takes the wedged relay out of its relay map and homes on another configured +relay in place — same node id, same sockets, nothing torn down — so clients off +the LAN (the mobile apps) are not stranded with connect timeouts until someone +restarts the service. The relay is put back once it is connectable again. See +[`docs/Architecture.md`](docs/Architecture.md#relay-failover-server-custom-relays). ## Relay and Address Lookup @@ -751,10 +751,16 @@ The short version as it applies to `ezvpn`: attaches to the connection as dial hints. These are **required** for connectivity in that mode — with lookup off there is no published record to fall back on — so configure both sides with the full relay list. -- Every configured custom relay is probed individually at startup and **all** - must come online, so a dead backup relay fails startup instead of hiding until - you need it. `relay_auth_token` (custom relays only) is validated by the same - probe. +- A custom relay set is **at least two distinct relays**: the server rides out + a relay outage by moving onto another configured relay, so one relay is + rejected at startup. +- Every configured custom relay is probed individually at startup. Startup + fails only when **none** comes online; a relay that does not is named in a + warning and left out of the relay map, so a relay that answers probes but + refuses connections cannot keep the process from ever coming online. The + server's failover puts it back once it is connectable; a client keeps it out + for its session. `relay_auth_token` (custom relays only) is validated by the + same probe. iroh address lookup is endpoint-ID resolution, not real/VPN DNS: it does not affect client DNS resolution, and the client does not push DNS or match domains diff --git a/docs/Architecture.md b/docs/Architecture.md index daa5b9e..2f004fa 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -702,54 +702,45 @@ sequenceDiagram - Jitter: 0-500ms added to prevent thundering herd - Counter reset: Resets to 0 after successful tunnel operation -### Relay Watchdog (Server, Custom Relays) +### Relay Failover (Server, Custom Relays) Implemented in the shared [flexaccess-iroh](https://github.com/flexaccessdev/flexaccess-iroh) crate -(`flexaccess_iroh::relay_watchdog`), driven by the serve loop in -`VpnServer::run`. A custom-relay server is dialable from off the LAN only while -it is **registered on its home relay** (n0 address lookup is off; clients dial -by relay hint, and a relay forwards QUIC Initials only to endpoints connected to -it). iroh has been observed to silently lose its home relay for good after a -routine relay reconnect (relays behind Cloudflare tunnels reset idle WebSockets -roughly hourly): no dial retries, no warnings, no registration on any relay — -the server stops being reachable through the relays until the process -restarts, while LAN clients that find it over mDNS keep working and hide the -outage. Relay-only clients see connect timeouts. - -The watchdog observes `Endpoint::home_relay_status()` and escalates like the -client's reconnect loop: - -1. no connected home relay for `RELAY_OUTAGE_NUDGE` (60s) → log a warning and - call `Endpoint::network_change()` (forces a fresh net report and relay - re-selection — enough when only the bookkeeping went stale); -2. still none at `RELAY_OUTAGE_REBUILD` (180s from the outage start) → the - serve loop closes the endpoint (bounded by `REBUILD_CLOSE_TIMEOUT`, 5s; a - slower close finishes in the background), binds a fresh one with the - **same identity** (`server_rebuild_factory`: no per-relay probe, online - wait tolerated failing), and accepts on it. The TUN device, address pools, - client registries, and status socket carry over; the old endpoint's - connections end with it and those clients reconnect on their own. The TUN - reader's self-encapsulation filter (the endpoint's local UDP ports) is - re-read for the fresh sockets. A failed rebuild is retried every - `REBUILD_RETRY` (30s). - -A rebuild only helps when iroh's bookkeeping went stale; when the relay itself -is unreachable the fresh endpoint never registers either, and rebuilding again -every three minutes would keep dropping the LAN clients that still work. The -watchdog therefore reports whether the endpoint held a home relay at any point -(`RelayOutage::relay_seen`), and the serve loop doubles the rebuild deadline -for each consecutive endpoint that never did (`rebuild_deadline`: 180s, 6m, -12m, 24m, then capped at `REBUILD_DEADLINE_MAX`, 30m). An endpoint that -registers resets the escalation to the usual 180s. The 60s nudge is unaffected. +(`flexaccess_iroh::relay_failover::fail_over_home_relay`), run by +`VpnServer::run` alongside its accept loop; the design is documented once in +[iroh-common-architecture/relay-failover.md](https://github.com/flexaccessdev/iroh-common-architecture/blob/main/relay-failover.md). +A custom-relay server is dialable from off the LAN only while it is +**registered on its home relay** (n0 address lookup is off; clients dial by +relay hint, and a relay forwards QUIC Initials only to endpoints connected to +it). iroh re-homes on its own when a relay is really down, but not when the +relay keeps answering net-report probes while relay connections to it fail +(relays behind Cloudflare tunnels that reset idle WebSockets): net_report keeps +preferring it, the relay connection never re-establishes, and the server is +registered nowhere until the process restarts. + +The failover watches `Endpoint::home_relay_status()`. After 60 s +(`RELAY_OUTAGE_FAILOVER`) without a connected home relay it takes the wedged +relay **out of the endpoint's relay map**; the forced net report can only +prefer a relay still in the map, so the endpoint homes on another configured +relay **in place**: same node id, same sockets, same direct paths, same +established connections, same TUN device. Nothing is rebuilt. The removed +relay is probed every 90 s (`RELAY_RESTORE_INTERVAL`) and put back once it is +connectable again. Relays that failed the **startup** probe are handled the +same way: the endpoint is bound without them (`CreatedEndpoint::relays_left_out`, +which `run` receives) and the failover restores them, so a process that +starts during such an outage still comes online on the relay that works. A reconnect at any point resets the outage clock. Non-home relays are connected on demand and dropped after a minute idle, which is normal and never counts as -an outage. With the default relays the watchdog is not armed: reachability -there rests on n0 publishing/resolution, not on one relay registration. - -The watchdog is shared with flextunnel through that crate: fix it there, tag a -release, and bump the tag here. +an outage. With the default relays the failover is pending forever: +reachability there rests on n0 publishing/resolution, not on one relay +registration. A custom relay set must therefore hold at least two distinct +relays, which `RelayConfig` enforces at startup. + +The failover is shared with tunnel-rs and flextunnel through that crate: fix it +there, tag a release, and bump the tag here. Until flexaccess-iroh v0.0.7 this +was a watchdog that rebuilt the endpoint (dropping every connection) after a +`network_change()` nudge that did nothing on a stable host; both are gone. ### Client Network Consistency Check (Reconnect) diff --git a/src/config/file_config.rs b/src/config/file_config.rs index 07c1d0e..c04f000 100644 --- a/src/config/file_config.rs +++ b/src/config/file_config.rs @@ -48,6 +48,9 @@ pub struct ServerAuthConfig { #[serde(deny_unknown_fields)] pub struct VpnServerIrohConfig { pub secret_file: Option, + /// Custom relay server URLs. At least two distinct relays: a server keeps + /// working through a relay outage by moving onto another configured relay, + /// so one relay is rejected. Both sides must list the full set. pub relay_urls: Option>, /// Optional shared bearer token sent to the custom relays as /// `Authorization: Bearer `. Only valid together with `relay_urls`. @@ -77,6 +80,9 @@ pub struct ClientAuthConfig { #[serde(deny_unknown_fields)] pub struct VpnClientIrohConfig { pub server_node_id: Option, + /// Custom relay server URLs. At least two distinct relays: a server keeps + /// working through a relay outage by moving onto another configured relay, + /// so one relay is rejected. Both sides must list the full set. pub relay_urls: Option>, /// Optional shared bearer token sent to the custom relays as /// `Authorization: Bearer `. Only valid together with `relay_urls`. @@ -629,7 +635,7 @@ auth_key_file = "./client.key" [iroh] server_node_id = "2xnbkpbc7izsilvewd7c62w7wnwziacmpfwvhcrya5nt76dqkpga" -relay_urls = ["https://relay.example.com"] +relay_urls = ["https://relay.example.com", "https://relay2.example.com"] "#, ) .unwrap(); @@ -640,7 +646,12 @@ relay_urls = ["https://relay.example.com"] let iroh = config.iroh.as_ref().unwrap(); assert_eq!( iroh.relay_urls.as_deref(), - Some(&["https://relay.example.com".to_string()][..]) + Some( + &[ + "https://relay.example.com".to_string(), + "https://relay2.example.com".to_string() + ][..] + ) ); let resolved = VpnClientConfigBuilder::new() @@ -650,9 +661,13 @@ relay_urls = ["https://relay.example.com"] .unwrap(); assert_eq!( resolved.relay_config, - RelayConfig::from_urls(&["https://relay.example.com".to_string()]).unwrap() + RelayConfig::from_urls(&[ + "https://relay.example.com".to_string(), + "https://relay2.example.com".to_string() + ]) + .unwrap() ); - assert_eq!(resolved.relay_config.custom_urls().len(), 1); + assert_eq!(resolved.relay_config.custom_urls().len(), 2); } #[test] @@ -663,7 +678,7 @@ role = "vpnclient" [iroh] server_node_id = "2xnbkpbc7izsilvewd7c62w7wnwziacmpfwvhcrya5nt76dqkpga" -relay_urls = ["https://relay.example.com"] +relay_urls = ["https://relay.example.com", "https://relay2.example.com"] relay_auth_token = "shared-secret" "#, ) diff --git a/src/main.rs b/src/main.rs index c98222a..d2402cf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,9 +25,8 @@ use ezvpn::config::file_config::{ load_vpn_client_config, load_vpn_server_config, }; use ezvpn::runtime::LockRole; -use ezvpn::transport::endpoint::{ - create_client_endpoint, create_server_endpoint, load_secret, server_rebuild_factory, -}; +use ezvpn::transport::endpoint::{create_client_endpoint, create_server_endpoint, load_secret}; +use flexaccess_iroh::endpoint::CreatedEndpoint; use ezvpn::transport::{ CongestionConfig, CongestionControl, parse_congestion_initial_window, set_congestion_config, }; @@ -182,7 +181,8 @@ enum ClientAction { #[arg(short = 'n', long)] server_node_id: Option, - /// Custom relay server URL(s) + /// Custom relay server URLs (at least two, for failover); the server + /// moves onto another one when its home relay is lost #[arg(long = "relay-url")] relay_urls: Vec, @@ -953,17 +953,10 @@ async fn run_vpn_server(resolved: ResolvedVpnServerConfig) -> Result<()> { // for VPN traffic; relays are only the automatic fallback when a direct // connection fails. A single endpoint serves both relay modes; internet // discovery follows the mode (on for default relays, off for custom). - let endpoint = create_server_endpoint(&resolved.relay_config, secret_key.clone()) - .await - .context("Failed to create iroh endpoint")?; - // The relay watchdog's remedy of last resort: a fresh endpoint with the - // same identity. Only a custom-relay server hangs its reachability on one - // home-relay registration (n0 discovery is off, clients dial by relay - // hint), so the watchdog is armed for custom relays only. - let rebuild = resolved - .relay_config - .is_custom() - .then(|| server_rebuild_factory(resolved.relay_config.clone(), secret_key)); + let CreatedEndpoint { endpoint, relays_left_out } = + create_server_endpoint(&resolved.relay_config, secret_key) + .await + .context("Failed to create iroh endpoint")?; log::info!("VPN Server Node ID: {}", endpoint.id()); log::info!( @@ -977,7 +970,7 @@ async fn run_vpn_server(resolved: ResolvedVpnServerConfig) -> Result<()> { .context("Failed to create VPN server")?; server - .run(endpoint, rebuild) + .run(endpoint, &resolved.relay_config, &relays_left_out) .await .map_err(|e| anyhow::anyhow!("VPN server error: {}", e)) } diff --git a/src/transport/endpoint.rs b/src/transport/endpoint.rs index 933879d..f21a8ae 100644 --- a/src/transport/endpoint.rs +++ b/src/transport/endpoint.rs @@ -2,7 +2,7 @@ //! [`flexaccess_iroh::endpoint`] builder — the VPN ALPN, its QUIC transport //! tuning, the client/server identity rules, the bounded connect, and the //! server's secret-key file. Relay configuration, the per-relay startup probe, -//! and the creation-vs-rebuild policy come from the shared crate. +//! and the bind-and-come-online policy come from the shared crate. use crate::error::{VpnError, VpnResult}; use crate::transport::build_quic_transport_config; @@ -10,17 +10,15 @@ use crate::tunnel::signaling::VPN_ALPN; use anyhow::{Context, Result}; use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; use flexaccess_iroh::endpoint::{ - EndpointOptions, create_endpoint, endpoint_builder, rebuild_endpoint, + CreatedEndpoint, EndpointOptions, create_endpoint, endpoint_builder, }; use iroh::{ Endpoint, EndpointAddr, EndpointId, SecretKey, endpoint::{Builder as EndpointBuilder, Connection}, }; use std::path::Path; -use std::sync::Arc; use std::time::Duration; -pub use flexaccess_iroh::endpoint::EndpointFactory; pub use flexaccess_iroh::relay::RelayConfig; /// Deadline for establishing the QUIC connection to the VPN server. @@ -96,9 +94,7 @@ fn base_builder(relay_config: &RelayConfig, publish_address: bool) -> Result Result { Ok(base_builder(relay_config, true)? .alpns(vec![VPN_ALPN.to_vec()]) @@ -112,30 +108,25 @@ fn server_builder(relay_config: &RelayConfig, secret: SecretKey) -> Result Result { +/// `VpnClient::resolve_server_addr`). Every custom relay is probed (startup +/// fails only if none answers), the endpoint is bound without the relays that +/// failed and must come online. The relays left out come back in +/// [`CreatedEndpoint::relays_left_out`] for the home-relay failover +/// (`VpnServer::run`) to restore once they are connectable. +pub async fn create_server_endpoint( + relay_config: &RelayConfig, + secret: SecretKey, +) -> Result { create_endpoint(relay_config, server_builder(relay_config, secret)?).await } -/// The rebuild recipe for the server endpoint, used when the relay watchdog -/// (`flexaccess_iroh::relay_watchdog`) gives up on the current one. Same -/// identity as the original, so the server's node id — what clients dial — -/// never changes. Tolerant rebuild policy (see [`rebuild_endpoint`]): no -/// relay probe, and the online wait may fail — the watchdog trips again if the -/// relays stay unreachable, with a lengthening deadline so a dead relay does -/// not churn the endpoint every few minutes (see `VpnServer::run`). -pub fn server_rebuild_factory(relay_config: RelayConfig, secret: SecretKey) -> EndpointFactory { - Arc::new(move || { - let relay_config = relay_config.clone(); - let secret = secret.clone(); - Box::pin(async move { rebuild_endpoint(server_builder(&relay_config, secret)?).await }) - }) -} - /// Create a client endpoint: ephemeral identity, never published (the client /// only dials out; its credential is the auth keypair, not the endpoint id). -/// Strict first-creation policy. +/// Same relay probe and online wait as the server. A relay that failed the +/// probe stays out for the client's lifetime: it lives one session and runs +/// no failover. pub async fn create_client_endpoint(relay_config: &RelayConfig) -> Result { - create_endpoint(relay_config, base_builder(relay_config, false)?).await + let CreatedEndpoint { endpoint, .. } = + create_endpoint(relay_config, base_builder(relay_config, false)?).await?; + Ok(endpoint) } diff --git a/src/tunnel/server.rs b/src/tunnel/server.rs index 0a950b2..61d2287 100644 --- a/src/tunnel/server.rs +++ b/src/tunnel/server.rs @@ -17,8 +17,8 @@ use crate::runtime::{LockRole, VpnLock}; use crate::tunnel::offload::VirtioNetHdr; use crate::transport::paths::{format_connection_paths, watch_connection_paths}; use crate::transport::SERVER_ADDR_PUBLISH_INTERVAL; -use crate::transport::endpoint::EndpointFactory; -use flexaccess_iroh::relay_watchdog::{self, RelayOutage}; +use crate::transport::endpoint::RelayConfig; +use flexaccess_iroh::relay_failover::fail_over_home_relay; use crate::tunnel::signaling::{ ClientAuthPayload, MAX_HANDSHAKE_SIZE, ServerAddrsMsg, VpnHandshake, VpnHandshakeResponse, read_message, write_message, @@ -28,15 +28,15 @@ use dashmap::DashMap; use futures::StreamExt; use ipnet::{Ipv4Net, Ipv6Net}; use iroh::endpoint::{Connection, RecvStream, SendStream}; -use iroh::{Endpoint, EndpointId, Watcher}; +use iroh::{Endpoint, EndpointId, RelayUrl, Watcher}; use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::sync::Arc; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; -use std::time::{Duration, Instant}; +use std::time::Instant; use tokio::io::ReadBuf; -use tokio::sync::{RwLock, mpsc, oneshot, watch}; +use tokio::sync::{RwLock, mpsc, oneshot}; /// Maximum number of frames drained from a channel per batch. const WRITE_BATCH_SIZE: usize = 256; @@ -756,25 +756,21 @@ impl VpnServer { /// Run the VPN server, accepting connections via iroh. /// - /// `rebuild`, when given, arms the home-relay watchdog - /// (`flexaccess_iroh::relay_watchdog`): if the endpoint loses its home relay for - /// good, the wedged endpoint is closed and `rebuild` binds its replacement - /// (same identity), which the server then serves on. Pass `None` for the - /// default relays, where reachability does not hang on one relay - /// registration. + /// With custom relays the shared home-relay failover + /// (`flexaccess_iroh::relay_failover`) runs alongside the accept loop and + /// keeps the server dialable: `relay_config` is what it moves between, and + /// `relays_left_out` are the configured relays the endpoint was bound + /// without because they failed the startup probe, which it puts back once + /// they are connectable. With the default relays it is pending forever. pub async fn run( mut self, endpoint: Endpoint, - rebuild: Option, + relay_config: &RelayConfig, + relays_left_out: &[RelayUrl], ) -> VpnResult<()> { // Setup TUN device self.setup_tun().await?; - // The live endpoint. Swapped by the relay watchdog's rebuild; the TUN - // reader (its self-encapsulation filter) and the status socket follow - // it through the receiver. - let (endpoint_tx, endpoint_rx) = watch::channel(endpoint.clone()); - log::info!("VPN Server started:"); // Log IPv4 info if configured if let Some(ref ip_pool) = self.ip_pool { @@ -873,12 +869,12 @@ impl VpnServer { let started_at = Instant::now(); let node_id = endpoint.id().to_string(); let status_server = server.clone(); - let status_endpoint_rx = endpoint_rx.clone(); + let status_endpoint = endpoint.clone(); let status_overlay_v4 = server.config.network; let status_overlay_v6 = server.config.network6; let _status_listener = crate::control::spawn_status_listener(LockRole::Server, "default", move || { - let status_endpoint = status_endpoint_rx.borrow().clone(); + let status_endpoint = status_endpoint.clone(); let status_server = status_server.clone(); let node_id = node_id.clone(); async move { @@ -906,78 +902,29 @@ impl VpnServer { // Spawn TUN reader task (reads from TUN, routes to clients) // Store JoinHandle for graceful shutdown. let server_tun = server.clone(); + let tun_endpoint = endpoint.clone(); let tun_reader_handle = tokio::spawn(async move { - if let Err(e) = server_tun.run_tun_reader(tun_reader, endpoint_rx).await { + if let Err(e) = server_tun.run_tun_reader(tun_reader, &tun_endpoint).await { log::error!("TUN reader error: {}", e); } }); - // Serve loop. A pass accepts on the current endpoint until it closes, - // or — custom relays only — until the relay watchdog reports the - // endpoint has lost its home relay for good. That last case is the - // in-process equivalent of the process restart known to fix it: close - // the wedged endpoint, bind a fresh one with the same identity, and - // serve again. The TUN device, address pools, and client registries - // carry over; the old endpoint's connections (and their handler - // tasks) end with it, and those clients reconnect on their own. - // - // A rebuild only helps when iroh's relay bookkeeping went stale. When - // the relay itself is unreachable the fresh endpoint never registers - // either, and rebuilding it again every few minutes would keep - // dropping the LAN clients that still work. So consecutive endpoints - // that never saw a home relay lengthen the watchdog's deadline - // (`rebuild_deadline`); one that did register resets the escalation. - let mut endpoint = endpoint; - let mut unregistered_endpoints: u32 = 0; - loop { - let deadline = rebuild_deadline(unregistered_endpoints); - let outage = tokio::select! { - () = server.accept_loop(&endpoint, &tun_write_tx) => { - log::info!("Endpoint closed, shutting down"); - break; - } - outage = watch_home_relay_if(rebuild.as_ref(), &endpoint, deadline) => outage, - }; - let rebuild = rebuild - .as_ref() - .expect("watchdog only runs with a rebuild recipe"); - - unregistered_endpoints = if outage.relay_seen { - 0 - } else { - unregistered_endpoints + 1 - }; - log::error!( - "No connected home relay for {:.0}s despite a network re-check; rebuilding the \ - endpoint from scratch (server node id stays {})", - outage.duration.as_secs_f64(), - endpoint.id() - ); - if unregistered_endpoints > 0 { - log::error!( - "{unregistered_endpoints} endpoint(s) in a row never registered on any home \ - relay; the relay itself is probably unreachable. If the rebuilt endpoint \ - does not register either, the next rebuild waits {}s", - rebuild_deadline(unregistered_endpoints).as_secs() - ); + // Accept until the endpoint closes. Alongside, with custom relays, the + // shared home-relay failover keeps the server dialable: a custom-relay + // server is reachable from off the LAN only through its home relay (n0 + // address lookup is off, clients dial by relay hint), and if that + // relay is lost for a minute without iroh re-homing on its own, the + // failover moves the endpoint onto another configured relay in place. + // Nothing is torn down: the identity, the direct paths, the + // established connections and the TUN device all stay. Relays the + // startup probe could not connect (the endpoint was bound without + // them) are put back by the same failover once they are. With the + // default relays the failover future is pending forever. + tokio::select! { + () = server.accept_loop(&endpoint, &tun_write_tx) => { + log::info!("Endpoint closed, shutting down"); } - close_endpoint_bounded(endpoint).await; - endpoint = loop { - match rebuild().await { - Ok(fresh) => break fresh, - Err(e) => { - log::error!( - "Endpoint rebuild failed: {e:#}; retrying in {}s", - REBUILD_RETRY.as_secs() - ); - tokio::time::sleep(REBUILD_RETRY).await; - } - } - }; - // `send` fails only when every receiver is gone; the TUN reader - // holds one for the life of `run`. - let _ = endpoint_tx.send(endpoint.clone()); - log::warn!("Endpoint rebuilt; serving again as {}", endpoint.id()); + () = fail_over_home_relay(&endpoint, relay_config, relays_left_out) => {} } // Graceful shutdown: drop channel sender to signal TUN writer to exit, @@ -1790,17 +1737,17 @@ impl VpnServer { /// super-frames are software-segmented into per-MSS datagrams). Software GRO /// coalescing is gone: a datagram can only carry a single MTU-sized packet. /// - /// `endpoint_rx` carries the live endpoint; its local UDP port set (the - /// self-encapsulation filter) is re-read whenever the relay watchdog - /// rebuilds the endpoint, which binds fresh sockets on fresh ports. + /// `endpoint`'s local UDP port set is the self-encapsulation filter; the + /// endpoint is bound once for the life of `run` (a relay failover moves + /// its home relay, not its sockets), so the set is read once. async fn run_tun_reader( &self, mut tun_reader: crate::net::device::TunReader, - mut endpoint_rx: watch::Receiver, + endpoint: &Endpoint, ) -> VpnResult<()> { log::info!("TUN reader started"); - let mut local_iroh_udp_ports = collect_local_iroh_udp_ports(&endpoint_rx.borrow_and_update()); + let local_iroh_udp_ports = collect_local_iroh_udp_ports(endpoint); let buffer_size = tun_reader.buffer_size(); let mut read_storage = uninitialized_vec(buffer_size); @@ -1829,18 +1776,6 @@ impl VpnServer { let raw_frame = packet_buf.filled(); self.stats.tun_packets_read.fetch_add(1, Ordering::Relaxed); - // One atomic load per packet; the set itself is only rebuilt when - // the endpoint was. `Err` means the sender is gone (`run` is - // ending), which changes nothing here. - if endpoint_rx.has_changed().unwrap_or(false) { - local_iroh_udp_ports = - collect_local_iroh_udp_ports(&endpoint_rx.borrow_and_update()); - log::info!( - "Filtering tunneled traffic for {} local iroh UDP port(s) of the rebuilt endpoint", - local_iroh_udp_ports.len() - ); - } - let (offload, packet_ref) = match tun_reader.split_frame(raw_frame) { Ok(parts) => parts, Err(e) => { @@ -2044,61 +1979,6 @@ fn collect_local_iroh_udp_ports(endpoint: &Endpoint) -> HashSet { endpoint.addr().ip_addrs().map(|addr| addr.port()).collect() } -/// Pause between attempts to bind a replacement endpoint after the relay -/// watchdog retired the old one and the rebuild itself failed (e.g. no route -/// to bind on). The server has no endpoint at all during this wait, so it is -/// short — there is nothing to lose by trying again soon. -const REBUILD_RETRY: Duration = Duration::from_secs(30); - -/// Cap on the watchdog's rebuild deadline once consecutive rebuilt endpoints -/// keep failing to register on any home relay. -const REBUILD_DEADLINE_MAX: Duration = Duration::from_secs(30 * 60); - -/// The watchdog's rebuild deadline for the next serve pass, given how many -/// endpoints in a row never registered on a home relay: the usual -/// [`relay_watchdog::RELAY_OUTAGE_REBUILD`] after an endpoint that did -/// register, doubling per unregistered endpoint up to [`REBUILD_DEADLINE_MAX`] -/// (180s, 6m, 12m, 24m, 30m). Rebuilding while the relay itself is down -/// gains nothing and drops every LAN client, so it is done less and less -/// often; a relay that comes back resets the escalation. -fn rebuild_deadline(unregistered_endpoints: u32) -> Duration { - let factor = 1u32 << unregistered_endpoints.min(4); - (relay_watchdog::RELAY_OUTAGE_REBUILD * factor).min(REBUILD_DEADLINE_MAX) -} - -/// Bound wait on the old endpoint's graceful close during a rebuild. The close -/// runs as its own task and is never cancelled (dropping a bound endpoint -/// without `close()` is fatal under the release profile's panic=abort); the -/// bound only keeps the serve loop from stalling behind it, letting a slow -/// close finish in the background while the replacement is bound. -const REBUILD_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); - -/// The relay watchdog for one serve pass: pending forever when no rebuild -/// recipe was given (default relays), otherwise resolves once the endpoint -/// has had no home relay for `rebuild_after` and should be replaced. -async fn watch_home_relay_if( - rebuild: Option<&EndpointFactory>, - endpoint: &Endpoint, - rebuild_after: Duration, -) -> RelayOutage { - match rebuild { - Some(_) => relay_watchdog::watch_home_relay(endpoint, rebuild_after).await, - None => std::future::pending().await, - } -} - -/// Close a retired endpoint, waiting at most [`REBUILD_CLOSE_TIMEOUT`] for it; -/// a slower close carries on in its own task. -async fn close_endpoint_bounded(endpoint: Endpoint) { - let close = tokio::spawn(async move { endpoint.close().await }); - if tokio::time::timeout(REBUILD_CLOSE_TIMEOUT, close).await.is_err() { - log::warn!( - "Old endpoint is taking more than {}s to close; continuing in the background", - REBUILD_CLOSE_TIMEOUT.as_secs() - ); - } -} - /// The server's candidate iroh underlay addresses (deduped, sorted): the set it /// advertises to clients for bypass routing, in the handshake and over the data /// path. Shared by the handshake response, the periodic publisher, and `status`. @@ -2292,17 +2172,6 @@ fn extract_udp_ports(packet: &[u8]) -> Option<(u16, u16)> { mod tests { use super::*; - #[test] - fn rebuild_deadline_doubles_per_unregistered_endpoint_up_to_the_cap() { - let base = relay_watchdog::RELAY_OUTAGE_REBUILD; - assert_eq!(rebuild_deadline(0), base); - assert_eq!(rebuild_deadline(1), base * 2); - assert_eq!(rebuild_deadline(2), base * 4); - assert_eq!(rebuild_deadline(3), base * 8); - assert_eq!(rebuild_deadline(4), REBUILD_DEADLINE_MAX); - assert_eq!(rebuild_deadline(50), REBUILD_DEADLINE_MAX); - } - /// Helper to create a random EndpointId for testing fn random_endpoint_id() -> EndpointId { let bytes: [u8; 32] = rand::random();