From 209dc11a4ecf84283bf5cac7129b89f516878f35 Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Sat, 5 Sep 2026 09:36:24 -0700 Subject: [PATCH 1/2] Replace the home-relay watchdog with in-place relay failover (flexaccess-iroh v0.0.8) Adopt flexaccess-iroh v0.0.8. The watchdog (a 60 s network_change() nudge that is a no-op on a stable host and a 180 s endpoint rebuild that dropped every connection) is gone; `VpnServer::run` runs `relay_failover::fail_over_home_relay` alongside its accept loop: after 60 s without a connected home relay the wedged relay is taken out of the relay map so iroh's forced net report homes the endpoint on another configured relay in place, and the relay is put back once a probe shows it connectable. v0.0.8 also binds the endpoint without the custom relays that fail the startup probe (a relay that answers probes but refuses relay connections would otherwise be preferred, never connect, and keep the process from ever coming online); `create_server_endpoint` returns `CreatedEndpoint` and the server hands `relays_left_out` to the failover to restore. The client keeps such a relay out for its session. Consequences for ezvpn: - custom relays (`relay_urls` / `--relay-url`) require at least two distinct URLs; default relay mode is unchanged - the endpoint watch channel that let the TUN reader and the status socket follow a rebuilt endpoint is gone: the endpoint is bound once for the life of `run`, so the self-encapsulation port filter is read once - `server_rebuild_factory`, `EndpointFactory` and the rebuild deadline escalation are removed Docs point at iroh-common-architecture's relay-failover.md. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01RZhxxjP2TA93PHNr6p1BU8 --- Cargo.lock | 4 +- Cargo.toml | 2 +- README.md | 26 +++-- docs/Architecture.md | 75 ++++++-------- src/config/file_config.rs | 25 ++++- src/main.rs | 25 ++--- src/transport/endpoint.rs | 45 ++++----- src/tunnel/server.rs | 205 +++++++------------------------------- 8 files changed, 136 insertions(+), 271 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d3ab40..b9f7b08 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -997,8 +997,8 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "flexaccess-iroh" -version = "0.0.3" -source = "git+https://github.com/flexaccessdev/flexaccess-iroh?tag=v0.0.3#422dacee5b38a667bd6da2b709d28d86c8d7661f" +version = "0.0.8" +source = "git+https://github.com/flexaccessdev/flexaccess-iroh?tag=v0.0.8#e718f5d0b00f9837a7bea50114d77320f31f5197" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index f45baa7..43ddb6c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,7 +34,7 @@ ipnet = { version = "2", features = ["serde"] } # Shared iroh transport layer (RelayConfig + per-relay probe, endpoint # build/rebuild, home-relay watchdog, endpoint-bound auth transcript). No # `mdns` feature: ezvpn runs no local-network discovery. -flexaccess-iroh = { git = "https://github.com/flexaccessdev/flexaccess-iroh", tag = "v0.0.3" } +flexaccess-iroh = { git = "https://github.com/flexaccessdev/flexaccess-iroh", tag = "v0.0.8" } # Redirected to the fork below via `[patch.crates-io]`, which also covers the # shared crate's own `iroh` dependency so the graph holds a single `iroh`. iroh = "1.1.0" diff --git a/README.md b/README.md index f4de76e..9d8ea2f 100644 --- a/README.md +++ b/README.md @@ -722,12 +722,12 @@ the same IP during reconnects. Reassignment is expected mainly after server restart or allocation state changes. A **server** with custom relays watches its own home-relay registration: if it -has no connected home relay for 60s it re-checks the network, and if that has -not helped by 180s it rebuilds its endpoint in place (same node id) — the -in-process equivalent of a restart, so relay-only clients (the mobile apps, -anything off the LAN) are not stranded with connect timeouts until someone -restarts the service. See -[`docs/Architecture.md`](docs/Architecture.md#relay-watchdog-server-custom-relays). +has no connected home relay for 60s and iroh has not re-homed it on its own, +it takes the wedged relay out of its relay map and homes on another configured +relay in place — same node id, same sockets, nothing torn down — so clients off +the LAN (the mobile apps) are not stranded with connect timeouts until someone +restarts the service. The relay is put back once it is connectable again. See +[`docs/Architecture.md`](docs/Architecture.md#relay-failover-server-custom-relays). ## Relay and Address Lookup @@ -751,10 +751,16 @@ The short version as it applies to `ezvpn`: attaches to the connection as dial hints. These are **required** for connectivity in that mode — with lookup off there is no published record to fall back on — so configure both sides with the full relay list. -- Every configured custom relay is probed individually at startup and **all** - must come online, so a dead backup relay fails startup instead of hiding until - you need it. `relay_auth_token` (custom relays only) is validated by the same - probe. +- A custom relay set is **at least two distinct relays**: the server rides out + a relay outage by moving onto another configured relay, so one relay is + rejected at startup. +- Every configured custom relay is probed individually at startup. Startup + fails only when **none** comes online; a relay that does not is named in a + warning and left out of the relay map, so a relay that answers probes but + refuses connections cannot keep the process from ever coming online. The + server's failover puts it back once it is connectable; a client keeps it out + for its session. `relay_auth_token` (custom relays only) is validated by the + same probe. iroh address lookup is endpoint-ID resolution, not real/VPN DNS: it does not affect client DNS resolution, and the client does not push DNS or match domains diff --git a/docs/Architecture.md b/docs/Architecture.md index daa5b9e..2f004fa 100644 --- a/docs/Architecture.md +++ b/docs/Architecture.md @@ -702,54 +702,45 @@ sequenceDiagram - Jitter: 0-500ms added to prevent thundering herd - Counter reset: Resets to 0 after successful tunnel operation -### Relay Watchdog (Server, Custom Relays) +### Relay Failover (Server, Custom Relays) Implemented in the shared [flexaccess-iroh](https://github.com/flexaccessdev/flexaccess-iroh) crate -(`flexaccess_iroh::relay_watchdog`), driven by the serve loop in -`VpnServer::run`. A custom-relay server is dialable from off the LAN only while -it is **registered on its home relay** (n0 address lookup is off; clients dial -by relay hint, and a relay forwards QUIC Initials only to endpoints connected to -it). iroh has been observed to silently lose its home relay for good after a -routine relay reconnect (relays behind Cloudflare tunnels reset idle WebSockets -roughly hourly): no dial retries, no warnings, no registration on any relay — -the server stops being reachable through the relays until the process -restarts, while LAN clients that find it over mDNS keep working and hide the -outage. Relay-only clients see connect timeouts. - -The watchdog observes `Endpoint::home_relay_status()` and escalates like the -client's reconnect loop: - -1. no connected home relay for `RELAY_OUTAGE_NUDGE` (60s) → log a warning and - call `Endpoint::network_change()` (forces a fresh net report and relay - re-selection — enough when only the bookkeeping went stale); -2. still none at `RELAY_OUTAGE_REBUILD` (180s from the outage start) → the - serve loop closes the endpoint (bounded by `REBUILD_CLOSE_TIMEOUT`, 5s; a - slower close finishes in the background), binds a fresh one with the - **same identity** (`server_rebuild_factory`: no per-relay probe, online - wait tolerated failing), and accepts on it. The TUN device, address pools, - client registries, and status socket carry over; the old endpoint's - connections end with it and those clients reconnect on their own. The TUN - reader's self-encapsulation filter (the endpoint's local UDP ports) is - re-read for the fresh sockets. A failed rebuild is retried every - `REBUILD_RETRY` (30s). - -A rebuild only helps when iroh's bookkeeping went stale; when the relay itself -is unreachable the fresh endpoint never registers either, and rebuilding again -every three minutes would keep dropping the LAN clients that still work. The -watchdog therefore reports whether the endpoint held a home relay at any point -(`RelayOutage::relay_seen`), and the serve loop doubles the rebuild deadline -for each consecutive endpoint that never did (`rebuild_deadline`: 180s, 6m, -12m, 24m, then capped at `REBUILD_DEADLINE_MAX`, 30m). An endpoint that -registers resets the escalation to the usual 180s. The 60s nudge is unaffected. +(`flexaccess_iroh::relay_failover::fail_over_home_relay`), run by +`VpnServer::run` alongside its accept loop; the design is documented once in +[iroh-common-architecture/relay-failover.md](https://github.com/flexaccessdev/iroh-common-architecture/blob/main/relay-failover.md). +A custom-relay server is dialable from off the LAN only while it is +**registered on its home relay** (n0 address lookup is off; clients dial by +relay hint, and a relay forwards QUIC Initials only to endpoints connected to +it). iroh re-homes on its own when a relay is really down, but not when the +relay keeps answering net-report probes while relay connections to it fail +(relays behind Cloudflare tunnels that reset idle WebSockets): net_report keeps +preferring it, the relay connection never re-establishes, and the server is +registered nowhere until the process restarts. + +The failover watches `Endpoint::home_relay_status()`. After 60 s +(`RELAY_OUTAGE_FAILOVER`) without a connected home relay it takes the wedged +relay **out of the endpoint's relay map**; the forced net report can only +prefer a relay still in the map, so the endpoint homes on another configured +relay **in place**: same node id, same sockets, same direct paths, same +established connections, same TUN device. Nothing is rebuilt. The removed +relay is probed every 90 s (`RELAY_RESTORE_INTERVAL`) and put back once it is +connectable again. Relays that failed the **startup** probe are handled the +same way: the endpoint is bound without them (`CreatedEndpoint::relays_left_out`, +which `run` receives) and the failover restores them, so a process that +starts during such an outage still comes online on the relay that works. A reconnect at any point resets the outage clock. Non-home relays are connected on demand and dropped after a minute idle, which is normal and never counts as -an outage. With the default relays the watchdog is not armed: reachability -there rests on n0 publishing/resolution, not on one relay registration. - -The watchdog is shared with flextunnel through that crate: fix it there, tag a -release, and bump the tag here. +an outage. With the default relays the failover is pending forever: +reachability there rests on n0 publishing/resolution, not on one relay +registration. A custom relay set must therefore hold at least two distinct +relays, which `RelayConfig` enforces at startup. + +The failover is shared with tunnel-rs and flextunnel through that crate: fix it +there, tag a release, and bump the tag here. Until flexaccess-iroh v0.0.7 this +was a watchdog that rebuilt the endpoint (dropping every connection) after a +`network_change()` nudge that did nothing on a stable host; both are gone. ### Client Network Consistency Check (Reconnect) diff --git a/src/config/file_config.rs b/src/config/file_config.rs index 07c1d0e..c04f000 100644 --- a/src/config/file_config.rs +++ b/src/config/file_config.rs @@ -48,6 +48,9 @@ pub struct ServerAuthConfig { #[serde(deny_unknown_fields)] pub struct VpnServerIrohConfig { pub secret_file: Option, + /// Custom relay server URLs. At least two distinct relays: a server keeps + /// working through a relay outage by moving onto another configured relay, + /// so one relay is rejected. Both sides must list the full set. pub relay_urls: Option>, /// Optional shared bearer token sent to the custom relays as /// `Authorization: Bearer `. Only valid together with `relay_urls`. @@ -77,6 +80,9 @@ pub struct ClientAuthConfig { #[serde(deny_unknown_fields)] pub struct VpnClientIrohConfig { pub server_node_id: Option, + /// Custom relay server URLs. At least two distinct relays: a server keeps + /// working through a relay outage by moving onto another configured relay, + /// so one relay is rejected. Both sides must list the full set. pub relay_urls: Option>, /// Optional shared bearer token sent to the custom relays as /// `Authorization: Bearer `. Only valid together with `relay_urls`. @@ -629,7 +635,7 @@ auth_key_file = "./client.key" [iroh] server_node_id = "2xnbkpbc7izsilvewd7c62w7wnwziacmpfwvhcrya5nt76dqkpga" -relay_urls = ["https://relay.example.com"] +relay_urls = ["https://relay.example.com", "https://relay2.example.com"] "#, ) .unwrap(); @@ -640,7 +646,12 @@ relay_urls = ["https://relay.example.com"] let iroh = config.iroh.as_ref().unwrap(); assert_eq!( iroh.relay_urls.as_deref(), - Some(&["https://relay.example.com".to_string()][..]) + Some( + &[ + "https://relay.example.com".to_string(), + "https://relay2.example.com".to_string() + ][..] + ) ); let resolved = VpnClientConfigBuilder::new() @@ -650,9 +661,13 @@ relay_urls = ["https://relay.example.com"] .unwrap(); assert_eq!( resolved.relay_config, - RelayConfig::from_urls(&["https://relay.example.com".to_string()]).unwrap() + RelayConfig::from_urls(&[ + "https://relay.example.com".to_string(), + "https://relay2.example.com".to_string() + ]) + .unwrap() ); - assert_eq!(resolved.relay_config.custom_urls().len(), 1); + assert_eq!(resolved.relay_config.custom_urls().len(), 2); } #[test] @@ -663,7 +678,7 @@ role = "vpnclient" [iroh] server_node_id = "2xnbkpbc7izsilvewd7c62w7wnwziacmpfwvhcrya5nt76dqkpga" -relay_urls = ["https://relay.example.com"] +relay_urls = ["https://relay.example.com", "https://relay2.example.com"] relay_auth_token = "shared-secret" "#, ) diff --git a/src/main.rs b/src/main.rs index c98222a..d2402cf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,9 +25,8 @@ use ezvpn::config::file_config::{ load_vpn_client_config, load_vpn_server_config, }; use ezvpn::runtime::LockRole; -use ezvpn::transport::endpoint::{ - create_client_endpoint, create_server_endpoint, load_secret, server_rebuild_factory, -}; +use ezvpn::transport::endpoint::{create_client_endpoint, create_server_endpoint, load_secret}; +use flexaccess_iroh::endpoint::CreatedEndpoint; use ezvpn::transport::{ CongestionConfig, CongestionControl, parse_congestion_initial_window, set_congestion_config, }; @@ -182,7 +181,8 @@ enum ClientAction { #[arg(short = 'n', long)] server_node_id: Option, - /// Custom relay server URL(s) + /// Custom relay server URLs (at least two, for failover); the server + /// moves onto another one when its home relay is lost #[arg(long = "relay-url")] relay_urls: Vec, @@ -953,17 +953,10 @@ async fn run_vpn_server(resolved: ResolvedVpnServerConfig) -> Result<()> { // for VPN traffic; relays are only the automatic fallback when a direct // connection fails. A single endpoint serves both relay modes; internet // discovery follows the mode (on for default relays, off for custom). - let endpoint = create_server_endpoint(&resolved.relay_config, secret_key.clone()) - .await - .context("Failed to create iroh endpoint")?; - // The relay watchdog's remedy of last resort: a fresh endpoint with the - // same identity. Only a custom-relay server hangs its reachability on one - // home-relay registration (n0 discovery is off, clients dial by relay - // hint), so the watchdog is armed for custom relays only. - let rebuild = resolved - .relay_config - .is_custom() - .then(|| server_rebuild_factory(resolved.relay_config.clone(), secret_key)); + let CreatedEndpoint { endpoint, relays_left_out } = + create_server_endpoint(&resolved.relay_config, secret_key) + .await + .context("Failed to create iroh endpoint")?; log::info!("VPN Server Node ID: {}", endpoint.id()); log::info!( @@ -977,7 +970,7 @@ async fn run_vpn_server(resolved: ResolvedVpnServerConfig) -> Result<()> { .context("Failed to create VPN server")?; server - .run(endpoint, rebuild) + .run(endpoint, &resolved.relay_config, &relays_left_out) .await .map_err(|e| anyhow::anyhow!("VPN server error: {}", e)) } diff --git a/src/transport/endpoint.rs b/src/transport/endpoint.rs index 933879d..f21a8ae 100644 --- a/src/transport/endpoint.rs +++ b/src/transport/endpoint.rs @@ -2,7 +2,7 @@ //! [`flexaccess_iroh::endpoint`] builder — the VPN ALPN, its QUIC transport //! tuning, the client/server identity rules, the bounded connect, and the //! server's secret-key file. Relay configuration, the per-relay startup probe, -//! and the creation-vs-rebuild policy come from the shared crate. +//! and the bind-and-come-online policy come from the shared crate. use crate::error::{VpnError, VpnResult}; use crate::transport::build_quic_transport_config; @@ -10,17 +10,15 @@ use crate::tunnel::signaling::VPN_ALPN; use anyhow::{Context, Result}; use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; use flexaccess_iroh::endpoint::{ - EndpointOptions, create_endpoint, endpoint_builder, rebuild_endpoint, + CreatedEndpoint, EndpointOptions, create_endpoint, endpoint_builder, }; use iroh::{ Endpoint, EndpointAddr, EndpointId, SecretKey, endpoint::{Builder as EndpointBuilder, Connection}, }; use std::path::Path; -use std::sync::Arc; use std::time::Duration; -pub use flexaccess_iroh::endpoint::EndpointFactory; pub use flexaccess_iroh::relay::RelayConfig; /// Deadline for establishing the QUIC connection to the VPN server. @@ -96,9 +94,7 @@ fn base_builder(relay_config: &RelayConfig, publish_address: bool) -> Result Result { Ok(base_builder(relay_config, true)? .alpns(vec![VPN_ALPN.to_vec()]) @@ -112,30 +108,25 @@ fn server_builder(relay_config: &RelayConfig, secret: SecretKey) -> Result Result { +/// `VpnClient::resolve_server_addr`). Every custom relay is probed (startup +/// fails only if none answers), the endpoint is bound without the relays that +/// failed and must come online. The relays left out come back in +/// [`CreatedEndpoint::relays_left_out`] for the home-relay failover +/// (`VpnServer::run`) to restore once they are connectable. +pub async fn create_server_endpoint( + relay_config: &RelayConfig, + secret: SecretKey, +) -> Result { create_endpoint(relay_config, server_builder(relay_config, secret)?).await } -/// The rebuild recipe for the server endpoint, used when the relay watchdog -/// (`flexaccess_iroh::relay_watchdog`) gives up on the current one. Same -/// identity as the original, so the server's node id — what clients dial — -/// never changes. Tolerant rebuild policy (see [`rebuild_endpoint`]): no -/// relay probe, and the online wait may fail — the watchdog trips again if the -/// relays stay unreachable, with a lengthening deadline so a dead relay does -/// not churn the endpoint every few minutes (see `VpnServer::run`). -pub fn server_rebuild_factory(relay_config: RelayConfig, secret: SecretKey) -> EndpointFactory { - Arc::new(move || { - let relay_config = relay_config.clone(); - let secret = secret.clone(); - Box::pin(async move { rebuild_endpoint(server_builder(&relay_config, secret)?).await }) - }) -} - /// Create a client endpoint: ephemeral identity, never published (the client /// only dials out; its credential is the auth keypair, not the endpoint id). -/// Strict first-creation policy. +/// Same relay probe and online wait as the server. A relay that failed the +/// probe stays out for the client's lifetime: it lives one session and runs +/// no failover. pub async fn create_client_endpoint(relay_config: &RelayConfig) -> Result { - create_endpoint(relay_config, base_builder(relay_config, false)?).await + let CreatedEndpoint { endpoint, .. } = + create_endpoint(relay_config, base_builder(relay_config, false)?).await?; + Ok(endpoint) } diff --git a/src/tunnel/server.rs b/src/tunnel/server.rs index 0a950b2..61d2287 100644 --- a/src/tunnel/server.rs +++ b/src/tunnel/server.rs @@ -17,8 +17,8 @@ use crate::runtime::{LockRole, VpnLock}; use crate::tunnel::offload::VirtioNetHdr; use crate::transport::paths::{format_connection_paths, watch_connection_paths}; use crate::transport::SERVER_ADDR_PUBLISH_INTERVAL; -use crate::transport::endpoint::EndpointFactory; -use flexaccess_iroh::relay_watchdog::{self, RelayOutage}; +use crate::transport::endpoint::RelayConfig; +use flexaccess_iroh::relay_failover::fail_over_home_relay; use crate::tunnel::signaling::{ ClientAuthPayload, MAX_HANDSHAKE_SIZE, ServerAddrsMsg, VpnHandshake, VpnHandshakeResponse, read_message, write_message, @@ -28,15 +28,15 @@ use dashmap::DashMap; use futures::StreamExt; use ipnet::{Ipv4Net, Ipv6Net}; use iroh::endpoint::{Connection, RecvStream, SendStream}; -use iroh::{Endpoint, EndpointId, Watcher}; +use iroh::{Endpoint, EndpointId, RelayUrl, Watcher}; use sha2::{Digest, Sha256}; use std::collections::{HashMap, HashSet}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use std::sync::Arc; use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; -use std::time::{Duration, Instant}; +use std::time::Instant; use tokio::io::ReadBuf; -use tokio::sync::{RwLock, mpsc, oneshot, watch}; +use tokio::sync::{RwLock, mpsc, oneshot}; /// Maximum number of frames drained from a channel per batch. const WRITE_BATCH_SIZE: usize = 256; @@ -756,25 +756,21 @@ impl VpnServer { /// Run the VPN server, accepting connections via iroh. /// - /// `rebuild`, when given, arms the home-relay watchdog - /// (`flexaccess_iroh::relay_watchdog`): if the endpoint loses its home relay for - /// good, the wedged endpoint is closed and `rebuild` binds its replacement - /// (same identity), which the server then serves on. Pass `None` for the - /// default relays, where reachability does not hang on one relay - /// registration. + /// With custom relays the shared home-relay failover + /// (`flexaccess_iroh::relay_failover`) runs alongside the accept loop and + /// keeps the server dialable: `relay_config` is what it moves between, and + /// `relays_left_out` are the configured relays the endpoint was bound + /// without because they failed the startup probe, which it puts back once + /// they are connectable. With the default relays it is pending forever. pub async fn run( mut self, endpoint: Endpoint, - rebuild: Option, + relay_config: &RelayConfig, + relays_left_out: &[RelayUrl], ) -> VpnResult<()> { // Setup TUN device self.setup_tun().await?; - // The live endpoint. Swapped by the relay watchdog's rebuild; the TUN - // reader (its self-encapsulation filter) and the status socket follow - // it through the receiver. - let (endpoint_tx, endpoint_rx) = watch::channel(endpoint.clone()); - log::info!("VPN Server started:"); // Log IPv4 info if configured if let Some(ref ip_pool) = self.ip_pool { @@ -873,12 +869,12 @@ impl VpnServer { let started_at = Instant::now(); let node_id = endpoint.id().to_string(); let status_server = server.clone(); - let status_endpoint_rx = endpoint_rx.clone(); + let status_endpoint = endpoint.clone(); let status_overlay_v4 = server.config.network; let status_overlay_v6 = server.config.network6; let _status_listener = crate::control::spawn_status_listener(LockRole::Server, "default", move || { - let status_endpoint = status_endpoint_rx.borrow().clone(); + let status_endpoint = status_endpoint.clone(); let status_server = status_server.clone(); let node_id = node_id.clone(); async move { @@ -906,78 +902,29 @@ impl VpnServer { // Spawn TUN reader task (reads from TUN, routes to clients) // Store JoinHandle for graceful shutdown. let server_tun = server.clone(); + let tun_endpoint = endpoint.clone(); let tun_reader_handle = tokio::spawn(async move { - if let Err(e) = server_tun.run_tun_reader(tun_reader, endpoint_rx).await { + if let Err(e) = server_tun.run_tun_reader(tun_reader, &tun_endpoint).await { log::error!("TUN reader error: {}", e); } }); - // Serve loop. A pass accepts on the current endpoint until it closes, - // or — custom relays only — until the relay watchdog reports the - // endpoint has lost its home relay for good. That last case is the - // in-process equivalent of the process restart known to fix it: close - // the wedged endpoint, bind a fresh one with the same identity, and - // serve again. The TUN device, address pools, and client registries - // carry over; the old endpoint's connections (and their handler - // tasks) end with it, and those clients reconnect on their own. - // - // A rebuild only helps when iroh's relay bookkeeping went stale. When - // the relay itself is unreachable the fresh endpoint never registers - // either, and rebuilding it again every few minutes would keep - // dropping the LAN clients that still work. So consecutive endpoints - // that never saw a home relay lengthen the watchdog's deadline - // (`rebuild_deadline`); one that did register resets the escalation. - let mut endpoint = endpoint; - let mut unregistered_endpoints: u32 = 0; - loop { - let deadline = rebuild_deadline(unregistered_endpoints); - let outage = tokio::select! { - () = server.accept_loop(&endpoint, &tun_write_tx) => { - log::info!("Endpoint closed, shutting down"); - break; - } - outage = watch_home_relay_if(rebuild.as_ref(), &endpoint, deadline) => outage, - }; - let rebuild = rebuild - .as_ref() - .expect("watchdog only runs with a rebuild recipe"); - - unregistered_endpoints = if outage.relay_seen { - 0 - } else { - unregistered_endpoints + 1 - }; - log::error!( - "No connected home relay for {:.0}s despite a network re-check; rebuilding the \ - endpoint from scratch (server node id stays {})", - outage.duration.as_secs_f64(), - endpoint.id() - ); - if unregistered_endpoints > 0 { - log::error!( - "{unregistered_endpoints} endpoint(s) in a row never registered on any home \ - relay; the relay itself is probably unreachable. If the rebuilt endpoint \ - does not register either, the next rebuild waits {}s", - rebuild_deadline(unregistered_endpoints).as_secs() - ); + // Accept until the endpoint closes. Alongside, with custom relays, the + // shared home-relay failover keeps the server dialable: a custom-relay + // server is reachable from off the LAN only through its home relay (n0 + // address lookup is off, clients dial by relay hint), and if that + // relay is lost for a minute without iroh re-homing on its own, the + // failover moves the endpoint onto another configured relay in place. + // Nothing is torn down: the identity, the direct paths, the + // established connections and the TUN device all stay. Relays the + // startup probe could not connect (the endpoint was bound without + // them) are put back by the same failover once they are. With the + // default relays the failover future is pending forever. + tokio::select! { + () = server.accept_loop(&endpoint, &tun_write_tx) => { + log::info!("Endpoint closed, shutting down"); } - close_endpoint_bounded(endpoint).await; - endpoint = loop { - match rebuild().await { - Ok(fresh) => break fresh, - Err(e) => { - log::error!( - "Endpoint rebuild failed: {e:#}; retrying in {}s", - REBUILD_RETRY.as_secs() - ); - tokio::time::sleep(REBUILD_RETRY).await; - } - } - }; - // `send` fails only when every receiver is gone; the TUN reader - // holds one for the life of `run`. - let _ = endpoint_tx.send(endpoint.clone()); - log::warn!("Endpoint rebuilt; serving again as {}", endpoint.id()); + () = fail_over_home_relay(&endpoint, relay_config, relays_left_out) => {} } // Graceful shutdown: drop channel sender to signal TUN writer to exit, @@ -1790,17 +1737,17 @@ impl VpnServer { /// super-frames are software-segmented into per-MSS datagrams). Software GRO /// coalescing is gone: a datagram can only carry a single MTU-sized packet. /// - /// `endpoint_rx` carries the live endpoint; its local UDP port set (the - /// self-encapsulation filter) is re-read whenever the relay watchdog - /// rebuilds the endpoint, which binds fresh sockets on fresh ports. + /// `endpoint`'s local UDP port set is the self-encapsulation filter; the + /// endpoint is bound once for the life of `run` (a relay failover moves + /// its home relay, not its sockets), so the set is read once. async fn run_tun_reader( &self, mut tun_reader: crate::net::device::TunReader, - mut endpoint_rx: watch::Receiver, + endpoint: &Endpoint, ) -> VpnResult<()> { log::info!("TUN reader started"); - let mut local_iroh_udp_ports = collect_local_iroh_udp_ports(&endpoint_rx.borrow_and_update()); + let local_iroh_udp_ports = collect_local_iroh_udp_ports(endpoint); let buffer_size = tun_reader.buffer_size(); let mut read_storage = uninitialized_vec(buffer_size); @@ -1829,18 +1776,6 @@ impl VpnServer { let raw_frame = packet_buf.filled(); self.stats.tun_packets_read.fetch_add(1, Ordering::Relaxed); - // One atomic load per packet; the set itself is only rebuilt when - // the endpoint was. `Err` means the sender is gone (`run` is - // ending), which changes nothing here. - if endpoint_rx.has_changed().unwrap_or(false) { - local_iroh_udp_ports = - collect_local_iroh_udp_ports(&endpoint_rx.borrow_and_update()); - log::info!( - "Filtering tunneled traffic for {} local iroh UDP port(s) of the rebuilt endpoint", - local_iroh_udp_ports.len() - ); - } - let (offload, packet_ref) = match tun_reader.split_frame(raw_frame) { Ok(parts) => parts, Err(e) => { @@ -2044,61 +1979,6 @@ fn collect_local_iroh_udp_ports(endpoint: &Endpoint) -> HashSet { endpoint.addr().ip_addrs().map(|addr| addr.port()).collect() } -/// Pause between attempts to bind a replacement endpoint after the relay -/// watchdog retired the old one and the rebuild itself failed (e.g. no route -/// to bind on). The server has no endpoint at all during this wait, so it is -/// short — there is nothing to lose by trying again soon. -const REBUILD_RETRY: Duration = Duration::from_secs(30); - -/// Cap on the watchdog's rebuild deadline once consecutive rebuilt endpoints -/// keep failing to register on any home relay. -const REBUILD_DEADLINE_MAX: Duration = Duration::from_secs(30 * 60); - -/// The watchdog's rebuild deadline for the next serve pass, given how many -/// endpoints in a row never registered on a home relay: the usual -/// [`relay_watchdog::RELAY_OUTAGE_REBUILD`] after an endpoint that did -/// register, doubling per unregistered endpoint up to [`REBUILD_DEADLINE_MAX`] -/// (180s, 6m, 12m, 24m, 30m). Rebuilding while the relay itself is down -/// gains nothing and drops every LAN client, so it is done less and less -/// often; a relay that comes back resets the escalation. -fn rebuild_deadline(unregistered_endpoints: u32) -> Duration { - let factor = 1u32 << unregistered_endpoints.min(4); - (relay_watchdog::RELAY_OUTAGE_REBUILD * factor).min(REBUILD_DEADLINE_MAX) -} - -/// Bound wait on the old endpoint's graceful close during a rebuild. The close -/// runs as its own task and is never cancelled (dropping a bound endpoint -/// without `close()` is fatal under the release profile's panic=abort); the -/// bound only keeps the serve loop from stalling behind it, letting a slow -/// close finish in the background while the replacement is bound. -const REBUILD_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); - -/// The relay watchdog for one serve pass: pending forever when no rebuild -/// recipe was given (default relays), otherwise resolves once the endpoint -/// has had no home relay for `rebuild_after` and should be replaced. -async fn watch_home_relay_if( - rebuild: Option<&EndpointFactory>, - endpoint: &Endpoint, - rebuild_after: Duration, -) -> RelayOutage { - match rebuild { - Some(_) => relay_watchdog::watch_home_relay(endpoint, rebuild_after).await, - None => std::future::pending().await, - } -} - -/// Close a retired endpoint, waiting at most [`REBUILD_CLOSE_TIMEOUT`] for it; -/// a slower close carries on in its own task. -async fn close_endpoint_bounded(endpoint: Endpoint) { - let close = tokio::spawn(async move { endpoint.close().await }); - if tokio::time::timeout(REBUILD_CLOSE_TIMEOUT, close).await.is_err() { - log::warn!( - "Old endpoint is taking more than {}s to close; continuing in the background", - REBUILD_CLOSE_TIMEOUT.as_secs() - ); - } -} - /// The server's candidate iroh underlay addresses (deduped, sorted): the set it /// advertises to clients for bypass routing, in the handshake and over the data /// path. Shared by the handshake response, the periodic publisher, and `status`. @@ -2292,17 +2172,6 @@ fn extract_udp_ports(packet: &[u8]) -> Option<(u16, u16)> { mod tests { use super::*; - #[test] - fn rebuild_deadline_doubles_per_unregistered_endpoint_up_to_the_cap() { - let base = relay_watchdog::RELAY_OUTAGE_REBUILD; - assert_eq!(rebuild_deadline(0), base); - assert_eq!(rebuild_deadline(1), base * 2); - assert_eq!(rebuild_deadline(2), base * 4); - assert_eq!(rebuild_deadline(3), base * 8); - assert_eq!(rebuild_deadline(4), REBUILD_DEADLINE_MAX); - assert_eq!(rebuild_deadline(50), REBUILD_DEADLINE_MAX); - } - /// Helper to create a random EndpointId for testing fn random_endpoint_id() -> EndpointId { let bytes: [u8; 32] = rand::random(); From 6f8cdf81b67300e6a222faf9bb69df433d6ee63d Mon Sep 17 00:00:00 2001 From: Andrew Chen Date: Sat, 5 Sep 2026 09:42:03 -0700 Subject: [PATCH 2/2] Bump version to 0.0.47 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01RZhxxjP2TA93PHNr6p1BU8 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b9f7b08..ec0b47f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -922,7 +922,7 @@ dependencies = [ [[package]] name = "ezvpn" -version = "0.0.46" +version = "0.0.47" dependencies = [ "android_logger", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 43ddb6c..31bfa99 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ezvpn" -version = "0.0.46" +version = "0.0.47" edition = "2024" description = "IP-over-QUIC VPN tunnel via iroh P2P" readme = "README.md"