diff --git a/crates/flextunnel-cli/src/client_session.rs b/crates/flextunnel-cli/src/client_session.rs index 507d264..a85e748 100644 --- a/crates/flextunnel-cli/src/client_session.rs +++ b/crates/flextunnel-cli/src/client_session.rs @@ -19,9 +19,7 @@ use flextunnel_core::forwards::{ }; use flextunnel_core::iroh::SecretKey; use flextunnel_core::proxy::{ClientAuth, ClientConfig, ProxyClient, reserved}; -use flextunnel_core::transport::endpoint::{ - RelayConfig, create_client_endpoint, create_quick_client_endpoint, -}; +use flextunnel_core::transport::endpoint::{ClientEndpoint, RelayConfig}; use flextunnel_core::transport::paths::{ConnPath, ConnPathKind}; use flextunnel_core::{app, auth, config}; @@ -157,7 +155,7 @@ pub async fn run_quick(r: config::ResolvedClient, client_secret: SecretKey) -> R /// endpoint, the proxy client and its live routes, the bound proxy listeners, /// the forward manager + set, and the status/mutation state. struct SessionRuntime { - endpoint: flextunnel_core::iroh::Endpoint, + endpoint: ClientEndpoint, client: std::sync::Arc, routes: std::sync::Arc>, socks_listener: Option, @@ -199,11 +197,11 @@ async fn build_session( .context("Invalid relay configuration")?; let (endpoint, auth) = match auth { SessionAuth::Key(client_key) => ( - create_client_endpoint(&relay_config).await, + ClientEndpoint::create(&relay_config).await, ClientAuth::Key(Box::new(client_key)), ), SessionAuth::Quick(secret) => ( - create_quick_client_endpoint(&relay_config, secret).await, + ClientEndpoint::create_quick(&relay_config, secret).await, ClientAuth::QuickAllowlisted, ), }; @@ -353,6 +351,10 @@ async fn drive_session( fwd_mgr.apply(&forwards); } state.observe_connection(routes.lock().map(|r| r.connected).unwrap_or(false)); + // The reconnect loop rebuilds the endpoint after repeated + // failures, which changes the (ephemeral) node id — keep the + // status display current. + state.client_node_id = endpoint.id().to_string(); } cmd = ipc_rx.recv() => match cmd { Some(IpcCmd::Status(reply)) => { @@ -420,7 +422,7 @@ async fn drive_session( } IpcSink::Panel(panel) => Some(panel), }; - crate::close_endpoint_or_exit(&endpoint).await; + crate::close_endpoint_or_exit(&endpoint.endpoint()).await; if let Some(panel) = panel { let _ = panel.await; } diff --git a/crates/flextunnel-core/src/proxy/bridge.rs b/crates/flextunnel-core/src/proxy/bridge.rs index 9254868..c8cb4a8 100644 --- a/crates/flextunnel-core/src/proxy/bridge.rs +++ b/crates/flextunnel-core/src/proxy/bridge.rs @@ -8,13 +8,16 @@ //! //! [`BRIDGE_ALPN`]: crate::transport::BRIDGE_ALPN //! -//! The connect/auth/heartbeat machinery mirrors [`super::client`], with one -//! deliberate difference in reconnect policy: a bridge retries **forever** (no +//! The connect/auth/heartbeat machinery mirrors [`super::client`], with two +//! deliberate differences in reconnect policy: a bridge retries **forever** (no //! fail-fast first connect, no attempt cap). The peer server may simply not be //! up yet, and a server daemon must not exit — or stop serving its other //! routes — because a peer is down. While the upstream is down, matching //! streams fail with host-unreachable (see `route_to_bridge` in -//! [`super::server`]). +//! [`super::server`]). And a bridge never escalates to the client's endpoint +//! rebuild: it dials on the **server's own endpoint**, which is also accepting +//! inbound clients on its persistent identity — rebuilding it to unwedge one +//! upstream would sever every connected client. use crate::error::{ProxyError, ProxyResult}; use crate::proxy::client::{calculate_backoff, client_heartbeat_loop, connect_with_timeout}; diff --git a/crates/flextunnel-core/src/proxy/client.rs b/crates/flextunnel-core/src/proxy/client.rs index 68f3025..475f5c3 100644 --- a/crates/flextunnel-core/src/proxy/client.rs +++ b/crates/flextunnel-core/src/proxy/client.rs @@ -5,7 +5,7 @@ use crate::error::{ProxyError, ProxyResult}; use crate::proxy::signaling::{self, ControlMsg, Hello, Target}; use crate::proxy::{dial, http, reserved, socks5, RoutedSet}; -use crate::transport::endpoint::RelayConfig; +use crate::transport::endpoint::{ClientEndpoint, RelayConfig}; use crate::transport::paths::{connection_paths, ConnPath, ConnectionSnapshot}; use crate::transport::{HEARTBEAT_INTERVAL, HEARTBEAT_INTERVAL_IDLE, liveness_window}; use anyhow::Result; @@ -27,6 +27,16 @@ use tokio::sync::{Semaphore, watch}; /// Reconnect backoff: base 1s, doubling per attempt, capped at 60s. const RECONNECT_BACKOFF_MAX: u64 = 60; +/// Escalate to a full endpoint rebuild every this many consecutive failed +/// reconnect attempts. The early attempts get the cheap `network_change()` +/// nudge, which repairs dead UDP sockets; a wedge that survives the nudge plus +/// two full connect timeouts is endpoint state a rebind cannot fix — a relay +/// link lost to a ping timeout and never re-established, stale cached paths +/// for the server — which only a fresh endpoint repairs (observed as "restart +/// the client process and it connects instantly"; the rebuild is that restart, +/// in-process). Rebuilding every Nth attempt (not once) keeps a long outage +/// retrying from fresh state without paying the rebuild on every backoff. +const REBUILD_ENDPOINT_ATTEMPTS: u32 = 3; /// Max jitter (ms) added to each backoff to avoid thundering reconnects. const RECONNECT_JITTER_MAX_MS: u64 = 500; /// Deadline for the server's handshake response. The QUIC keep-alive keeps the @@ -448,8 +458,9 @@ impl ProxyClient { /// `--no-auto-reconnect` is set). The listeners stay bound across reconnects: /// off-list targets keep connecting directly, while on-list requests are held /// for the reconnect (failing with network-unreachable only after - /// [`TUNNEL_RECOVERY_HOLD`]). - pub async fn run(&self, endpoint: &Endpoint) -> ProxyResult<()> { + /// [`TUNNEL_RECOVERY_HOLD`]). Reconnects that keep failing escalate to a + /// full endpoint rebuild every [`REBUILD_ENDPOINT_ATTEMPTS`] attempts. + pub async fn run(&self, endpoint: &ClientEndpoint) -> ProxyResult<()> { let socks = match self.config.socks_listen { Some(addr) => Some(TcpListener::bind(addr).await?), None => None, @@ -469,7 +480,7 @@ impl ProxyClient { /// This path never enables the HTTP front-end. pub async fn run_with_listener( &self, - endpoint: &Endpoint, + endpoint: &ClientEndpoint, listener: TcpListener, ) -> ProxyResult<()> { self.run_with_listeners(endpoint, listener, None).await @@ -483,7 +494,7 @@ impl ProxyClient { /// proxy listener. pub async fn run_with_listeners( &self, - endpoint: &Endpoint, + endpoint: &ClientEndpoint, socks_listener: TcpListener, http_listener: Option, ) -> ProxyResult<()> { @@ -499,7 +510,7 @@ impl ProxyClient { /// connection. Both may be absent for a forwarding-only GUI session. pub async fn run_with_optional_listeners( &self, - endpoint: &Endpoint, + endpoint: &ClientEndpoint, socks_listener: Option, http_listener: Option, ) -> ProxyResult<()> { @@ -526,7 +537,7 @@ impl ProxyClient { /// [`run_with_listeners`]. pub async fn run_with_listeners_ext( &self, - endpoint: &Endpoint, + endpoint: &ClientEndpoint, socks_listener: TcpListener, http_listener: Option, #[cfg(unix)] unix_listener: Option, @@ -543,7 +554,7 @@ impl ProxyClient { async fn run_with_optional_listeners_ext( &self, - endpoint: &Endpoint, + endpoint: &ClientEndpoint, socks_listener: Option, http_listener: Option, #[cfg(unix)] unix_listener: Option, @@ -628,7 +639,7 @@ impl ProxyClient { /// must succeed (fail fast); once connected, transient drops are retried. async fn manage_connection( &self, - endpoint: &Endpoint, + endpoint: &ClientEndpoint, current: &SharedConn, routed_set_shared: &SharedRoutedSet, ) -> ProxyResult<()> { @@ -644,20 +655,34 @@ impl ProxyClient { self.set_connected(false); current.send_replace(None); - // Retrying after a failure: the endpoint's UDP sockets may be dead - // underneath it (iOS defuncts them while the process is suspended; - // a sleeping laptop can do the same) and iroh cannot always detect - // that by itself, leaving reconnects wedged forever. Nudging it - // re-checks and rebinds the transports — harmless when nothing - // actually changed. - if attempt > 0 || path_returned { - endpoint.network_change().await; + if attempt > 0 && attempt.is_multiple_of(REBUILD_ENDPOINT_ATTEMPTS) { + // Escalation: the nudge below wasn't enough — rebuild the + // endpoint from scratch (see [`REBUILD_ENDPOINT_ATTEMPTS`]). + // On a rebuild failure (e.g. no route to bind on a dead + // network) the current endpoint stays in place and this + // attempt proceeds with it — the next multiple retries the + // rebuild. + log::warn!("Reconnect still failing after {attempt} attempts; rebuilding the endpoint from scratch"); + if let Err(e) = endpoint.rebuild().await { + log::warn!("Endpoint rebuild failed ({e:#}); retrying with the current endpoint"); + } + } else if attempt > 0 || path_returned { + // Retrying after a failure: the endpoint's UDP sockets may be dead + // underneath it (iOS defuncts them while the process is suspended; + // a sleeping laptop can do the same) and iroh cannot always detect + // that by itself, leaving reconnects wedged forever. Nudging it + // re-checks and rebinds the transports — harmless when nothing + // actually changed. + endpoint.endpoint().network_change().await; } // Establish (connect + auth). The handshake also learns the server's // tunnel set (drives split-tunneling) and returns the control-stream - // halves kept open for heartbeats. - let (connection, routed_set, ctrl_send, ctrl_recv) = match self.establish(endpoint).await + // halves kept open for heartbeats. The endpoint handle is taken + // fresh per attempt — a rebuild above swapped it. + let (connection, routed_set, ctrl_send, ctrl_recv) = match self + .establish(&endpoint.endpoint()) + .await { Ok(established) => { ever_connected = true; diff --git a/crates/flextunnel-core/src/proxy/e2e_tests.rs b/crates/flextunnel-core/src/proxy/e2e_tests.rs index 501bb5d..7a7a754 100644 --- a/crates/flextunnel-core/src/proxy/e2e_tests.rs +++ b/crates/flextunnel-core/src/proxy/e2e_tests.rs @@ -22,7 +22,7 @@ use crate::proxy::{ BridgeUpstream, BridgeUpstreamConfig, ClientAuth, ClientConfig, ForwardManager, ForwardSpec, ProxyClient, ProxyServer, ProxyServerParams, RoutedSet, ServerForwarder, }; -use crate::transport::endpoint::{AllowlistHook, EndpointAllowlists}; +use crate::transport::endpoint::{AllowlistHook, ClientEndpoint, EndpointAllowlists}; use crate::transport::{ALPN, BRIDGE_ALPN, QUICK_ALPN, build_quic_transport_config}; use iroh::address_lookup::MemoryLookup; use iroh::endpoint::{presets, Connection, RecvStream, SendStream}; @@ -465,6 +465,101 @@ async fn server_direct_forward_relays_and_server_rejects_off_list_target() { let _ = std::fs::remove_file(bl_path); } +/// Endpoint-rebuild escalation: when reconnect attempts keep failing on an +/// endpoint that is broken beyond repair (here: closed out from under the +/// session — the loopback stand-in for a wedged endpoint whose relay link or +/// path state never recovers), the reconnect loop must escalate to rebuilding +/// the endpoint via its factory and reconnect on the fresh one. The old +/// endpoint can never connect again, so recovery itself proves the rebuild +/// ran; the changed node id confirms it. +#[tokio::test] +async fn reconnect_rebuilds_a_dead_endpoint() { + let server_ep = loopback_endpoint(SecretKey::generate(), true).await; + let server_id = server_ep.id(); + let server_addr = EndpointAddr::new(server_id).with_ip_addr(server_ep.bound_sockets()[0]); + let bl_path = temp_blocklist("rebuild-endpoint"); + let (routed_set, routed_cidrs) = loopback_cidr_set(); + spawn_server_params( + server_ep.clone(), + ProxyServerParams { + routed_set, + routed_cidrs, + ..base_params(server_id, bl_path.clone()) + }, + ); + + let lookup = MemoryLookup::from_endpoint_info(vec![server_addr]); + let first_ep = loopback_endpoint_full( + SecretKey::generate(), + false, + lookup.clone(), + EndpointAllowlists::default(), + ) + .await; + let client_ep = ClientEndpoint::from_parts(first_ep.clone(), { + let lookup = lookup.clone(); + Arc::new(move || { + let lookup = lookup.clone(); + Box::pin(async move { + Ok(loopback_endpoint_full( + SecretKey::generate(), + false, + lookup, + EndpointAllowlists::default(), + ) + .await) + }) + }) + }); + let first_id = client_ep.id(); + + let client = Arc::new(ProxyClient::new(ClientConfig { + server_node_id: server_id.to_string(), + auth: ClientAuth::Key(Box::new(test_client_key().clone())), + socks_listen: None, + http_listen: None, + relay_urls: Vec::new(), + relay_auth_token: None, + auto_reconnect: true, + max_reconnect_attempts: None, + })); + let socks_listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + { + let (client, ep) = (client.clone(), client_ep.clone()); + tokio::spawn(async move { + if let Err(e) = client.run_with_listener(&ep, socks_listener).await { + eprintln!("e2e rebuild test client session ended: {e}"); + } + }); + } + let connected = || client.routes().lock().unwrap().connected; + wait_until("client to connect", connected).await; + + // Kill the client's own endpoint. Every reconnect attempt on it fails + // immediately, so the backoff series reaches the rebuild escalation well + // inside the wait window. + first_ep.close().await; + wait_until("client to notice the drop", || !connected()).await; + // Own, longer bound: reaching the rebuild takes the full early backoff + // series (1s + 2s + 4s plus jitter), which crowds `wait_until`'s 10s. + let start = Instant::now(); + while !connected() { + assert!( + start.elapsed() < Duration::from_secs(20), + "timed out waiting for client to reconnect on a rebuilt endpoint" + ); + tokio::time::sleep(Duration::from_millis(25)).await; + } + + assert_ne!( + client_ep.id(), + first_id, + "recovery must have swapped in a freshly built endpoint" + ); + + let _ = std::fs::remove_file(bl_path); +} + /// Deploy-style connection holding: a SOCKS request for an on-list target /// arriving while the tunnel link is down is *held* for the client's own /// reconnect and then proceeds transparently on the fresh connection, instead @@ -502,6 +597,24 @@ async fn on_list_request_is_held_across_reconnect() { EndpointAllowlists::default(), ) .await; + // Hermetic rebuild recipe: should the session escalate to an endpoint + // rebuild mid-test, the replacement is another loopback endpoint sharing + // the same externally-held lookup. + let client_ep = ClientEndpoint::from_parts(client_ep, { + let lookup = lookup.clone(); + Arc::new(move || { + let lookup = lookup.clone(); + Box::pin(async move { + Ok(loopback_endpoint_full( + SecretKey::generate(), + false, + lookup, + EndpointAllowlists::default(), + ) + .await) + }) + }) + }); // The full reconnecting client session with a real SOCKS front-end. let client = Arc::new(ProxyClient::new(ClientConfig { diff --git a/crates/flextunnel-core/src/transport/endpoint.rs b/crates/flextunnel-core/src/transport/endpoint.rs index 02d9bad..b8586ad 100644 --- a/crates/flextunnel-core/src/transport/endpoint.rs +++ b/crates/flextunnel-core/src/transport/endpoint.rs @@ -3,7 +3,7 @@ use crate::transport::{ALPN, BRIDGE_ALPN, QUICK_ALPN, build_quic_transport_config}; use anyhow::{Context, Result}; use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; -use futures::future::join_all; +use futures::future::{BoxFuture, join_all}; use iroh::{ Endpoint, EndpointId, RelayMap, RelayMode, RelayUrl, SecretKey, endpoint::{ @@ -446,52 +446,168 @@ pub async fn create_server_endpoint( .await .context("Failed to create iroh endpoint")?; - wait_online(&endpoint).await?; + if let Err(e) = wait_online(&endpoint).await { + // Close before propagating: dropping a bound endpoint without + // `close()` is fatal under the release profile's panic=abort. + endpoint.close().await; + return Err(e); + } Ok(endpoint) } -/// Create a client endpoint (ephemeral identity). -pub async fn create_client_endpoint(relay_config: &RelayConfig) -> Result { - create_client_endpoint_inner(relay_config, None).await -} - -/// Create a **quick-mode** client endpoint: same as [`create_client_endpoint`] -/// but bound to the given (session-ephemeral) `secret`, so the endpoint id the -/// user entered on the quick server is the id this endpoint presents in the -/// TLS handshake — that id is the quick client's sole credential. The identity -/// is still never published (the client only dials), so pkarr publishing stays -/// off exactly as for an anonymous client. -pub async fn create_quick_client_endpoint( - relay_config: &RelayConfig, - secret: SecretKey, -) -> Result { - create_client_endpoint_inner(relay_config, Some(secret)).await -} - -async fn create_client_endpoint_inner( +/// Bind a client endpoint: no relay probe, no online wait — callers layer +/// their own creation-vs-rebuild policy over this. +async fn bind_client_endpoint( relay_config: &RelayConfig, secret: Option, ) -> Result { - print_relay_status(relay_config); - - // Validate each custom relay individually (fail if any is unreachable); a - // no-op for the default relays. - probe_custom_relays(relay_config).await?; - // `None` for the builder's lookup decision even with a quick secret: a // client never publishes its address (it only dials out). let mut builder = create_endpoint_builder(relay_config, None)?; if let Some(secret) = secret { builder = builder.secret_key(secret); } + builder.bind().await.context("Failed to create iroh endpoint") +} - let endpoint = builder - .bind() - .await - .context("Failed to create iroh endpoint")?; +/// Recipe producing a fresh, fully bound client endpoint — how a +/// [`ClientEndpoint`] rebuilds itself mid-session. +pub type EndpointFactory = Arc BoxFuture<'static, Result> + Send + Sync>; - wait_online(&endpoint).await?; - Ok(endpoint) +/// Bound wait on the old endpoint's graceful close during a rebuild. The close +/// runs as its own task and is never cancelled (dropping a bound endpoint +/// without `close()` is fatal under panic=abort — see the CLI's +/// `close_endpoint_or_exit`); the bound only keeps the reconnect loop from +/// stalling behind it, letting a slow close finish in the background. +const REBUILD_CLOSE_TIMEOUT: Duration = Duration::from_secs(5); + +/// A client endpoint that can be **rebuilt** from scratch mid-session. +/// +/// `Endpoint::network_change()` re-binds dead UDP transports, but a wedged +/// endpoint can be broken beyond what a rebind repairs: a relay link lost to a +/// ping timeout that never re-establishes, stale cached paths for the peer, +/// dead discovery state. A process restart always recovers because it builds a +/// brand-new endpoint; [`Self::rebuild`] gives the reconnect loop that same +/// remedy in-process — fresh sockets, fresh relay connections, fresh discovery +/// — without dropping the bound proxy listeners or the control socket. +/// +/// The handle is `Clone` and shared: the reconnect loop escalates to +/// [`Self::rebuild`] after repeated failures, while the embedder logs +/// [`Self::id`] and [`Self::close`]s whatever endpoint is current at teardown. +#[derive(Clone)] +pub struct ClientEndpoint { + /// The live endpoint, swapped by [`Self::rebuild`]. Std lock: accessors + /// clone the handle out synchronously and never hold it across an await. + current: Arc>, + factory: EndpointFactory, +} + +impl ClientEndpoint { + /// Create a client endpoint (ephemeral identity). + pub async fn create(relay_config: &RelayConfig) -> Result { + Self::create_inner(relay_config, None).await + } + + /// Create a **quick-mode** client endpoint: same as [`Self::create`] but + /// bound to the given (session-ephemeral) `secret`, so the endpoint id the + /// user entered on the quick server is the id this endpoint presents in the + /// TLS handshake — that id is the quick client's sole credential. The + /// identity is still never published (the client only dials), so pkarr + /// publishing stays off exactly as for an anonymous client. + pub async fn create_quick(relay_config: &RelayConfig, secret: SecretKey) -> Result { + Self::create_inner(relay_config, Some(secret)).await + } + + async fn create_inner(relay_config: &RelayConfig, secret: Option) -> Result { + print_relay_status(relay_config); + + // Validate each custom relay individually (fail if any is unreachable); + // a no-op for the default relays. + probe_custom_relays(relay_config).await?; + + let endpoint = bind_client_endpoint(relay_config, secret.clone()).await?; + if let Err(e) = wait_online(&endpoint).await { + // Close before propagating: dropping a bound endpoint without + // `close()` is fatal under the release profile's panic=abort. + endpoint.close().await; + return Err(e); + } + Ok(Self::from_parts( + endpoint, + client_rebuild_factory(relay_config.clone(), secret), + )) + } + + /// Wrap an externally bound endpoint with a caller-supplied rebuild recipe + /// (tests bind hermetic loopback endpoints and rebuild them the same way). + pub fn from_parts(endpoint: Endpoint, factory: EndpointFactory) -> Self { + Self { + current: Arc::new(std::sync::RwLock::new(endpoint)), + factory, + } + } + + /// A clone of the current endpoint handle. Take it fresh per use: a handle + /// held across a [`Self::rebuild`] keeps pointing at the old, closed + /// endpoint. + pub fn endpoint(&self) -> Endpoint { + self.current.read().expect("client endpoint lock").clone() + } + + /// The current endpoint id. Changes on rebuild for an ephemeral (keypair) + /// client; stable for a quick client (fixed secret). + pub fn id(&self) -> EndpointId { + self.endpoint().id() + } + + /// Swap in a freshly built endpoint and close the old one. On error the + /// current endpoint stays in place, so the caller can simply retry with it. + pub async fn rebuild(&self) -> Result<()> { + let fresh = (self.factory)().await?; + let old = { + let mut current = self.current.write().expect("client endpoint lock"); + std::mem::replace(&mut *current, fresh) + }; + // Graceful close on its own task: bounded wait here, but the task is + // never cancelled (see [`REBUILD_CLOSE_TIMEOUT`]). + let mut close = tokio::task::spawn(async move { old.close().await }); + if tokio::time::timeout(REBUILD_CLOSE_TIMEOUT, &mut close) + .await + .is_err() + { + log::warn!("Old endpoint's close is slow; leaving it to finish in the background"); + } + info!("Endpoint rebuilt; client node ID: {}", self.id()); + Ok(()) + } + + /// Close the current endpoint gracefully (session teardown). + pub async fn close(&self) { + self.endpoint().close().await; + } +} + +/// The rebuild recipe for a real client endpoint. Differs from first creation +/// deliberately: +/// +/// - **No per-relay probe.** At creation the probe validates the configuration +/// (fail fast if *any* relay is down); during an outage that strictness +/// would block recovery through the one relay that still answers. +/// - **The online wait is tolerated failing.** Even fully offline, a fresh +/// endpoint's mDNS discovery can still reach a LAN server — and the endpoint +/// being replaced is known-bad anyway, so the swap can only help. +fn client_rebuild_factory(relay_config: RelayConfig, secret: Option) -> EndpointFactory { + Arc::new(move || { + let relay_config = relay_config.clone(); + let secret = secret.clone(); + Box::pin(async move { + let endpoint = bind_client_endpoint(&relay_config, secret).await?; + if let Err(e) = wait_online(&endpoint).await { + log::warn!("Rebuilt endpoint: {e:#}; continuing (local discovery may still work)"); + } + Ok(endpoint) + }) + }) } #[cfg(test)] diff --git a/crates/flextunnel-desktop/src/tunnel.rs b/crates/flextunnel-desktop/src/tunnel.rs index 69e1b1c..e2da661 100644 --- a/crates/flextunnel-desktop/src/tunnel.rs +++ b/crates/flextunnel-desktop/src/tunnel.rs @@ -11,7 +11,7 @@ use crate::config::Profile; use flextunnel_core::forwards::{ForwardManager, ForwardStatus, PortForward}; use flextunnel_core::proxy::{ClientAuth, ClientConfig, ProxyClient, TunnelRoutes}; -use flextunnel_core::transport::endpoint::{RelayConfig, create_client_endpoint}; +use flextunnel_core::transport::endpoint::{ClientEndpoint, RelayConfig}; use flextunnel_core::transport::paths::ConnectionSnapshot; use std::collections::HashMap; use std::net::SocketAddr; @@ -443,7 +443,7 @@ async fn run_session( }; let mut create = tokio::spawn({ let relay_config = relay_config.clone(); - async move { create_client_endpoint(&relay_config).await } + async move { ClientEndpoint::create(&relay_config).await } }); let endpoint = loop { tokio::select! { diff --git a/crates/flextunnel-ffi/src/lib.rs b/crates/flextunnel-ffi/src/lib.rs index adeb0e3..4301a92 100644 --- a/crates/flextunnel-ffi/src/lib.rs +++ b/crates/flextunnel-ffi/src/lib.rs @@ -78,7 +78,7 @@ use flextunnel_core::proxy::signaling::Target; use flextunnel_core::proxy::{ ClientAuth, ClientConfig, ForwardManager, ForwardSpec, ForwardState, ProxyClient, TunnelRoutes, }; -use flextunnel_core::transport::endpoint::{RelayConfig, create_client_endpoint}; +use flextunnel_core::transport::endpoint::{ClientEndpoint, RelayConfig}; use flextunnel_core::transport::paths::ConnPathKind; /// Process-global guard enforcing **at most one** running proxy instance. A @@ -92,7 +92,7 @@ static RUNNING: AtomicBool = AtomicBool::new(false); pub struct FlextunnelHandle { runtime: tokio::runtime::Runtime, /// Kept so [`flextunnel_stop`] can close it gracefully before drop. - endpoint: iroh::Endpoint, + endpoint: ClientEndpoint, /// The client driving the serve loop. Shared with the spawned `task` (which /// holds a clone) so status callers can snapshot its live iroh paths on /// demand via [`flextunnel_conn_path`]. @@ -326,7 +326,7 @@ fn start_inner(json: &str) -> Result<(FlextunnelHandle, String), String> { let client_key = flextunnel_core::auth::ClientKey::from_secret_str(cfg.auth_key.trim()) .map_err(|e| format!("invalid auth key: {e:#}"))?; let endpoint = runtime - .block_on(create_client_endpoint(&relay_config)) + .block_on(ClientEndpoint::create(&relay_config)) .map_err(|e| format!("failed to create iroh endpoint: {e}"))?; let client = Arc::new(ProxyClient::new(ClientConfig { diff --git a/docs/architecture.md b/docs/architecture.md index 454bc50..26b69a4 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -230,6 +230,16 @@ Implemented in `ProxyClient::run` / `handle_failure`: backoff + jitter** (1s → 60s), indefinitely, unless `--max-reconnect-attempts` caps it or `--no-auto-reconnect` disables it. - Permanent errors (`AuthenticationFailed` / `Config`) never retry. +- Every **third** consecutive failure escalates to a **full endpoint rebuild** + (`ClientEndpoint::rebuild`); the other retries nudge + `Endpoint::network_change()` (rebinds dead UDP sockets) instead. The rebuild + swaps in a freshly bound endpoint — new sockets, new + relay connections, fresh discovery — and closes the wedged one in the + background. This is the in-process equivalent of restarting the + client, for wedges a rebind can't fix (a relay link lost to a ping timeout + and never re-established, stale cached paths for the server). The rebuild + skips the startup per-relay probe and tolerates the online-wait failing, so + a partial outage never blocks recovery. - The local proxy listeners stay bound across reconnects. Off-list targets keep connecting directly; on-list requests are held for the reconnect — up to 45s (`TUNNEL_RECOVERY_HOLD`), deploy-style connection holding — and only then fail diff --git a/docs/systemd.md b/docs/systemd.md index a42b456..9182ff1 100644 --- a/docs/systemd.md +++ b/docs/systemd.md @@ -58,10 +58,13 @@ The client already supervises itself where it matters: - After the **first** successful connection, auto-reconnect (on by default) retries transient drops internally with exponential backoff, indefinitely. - The process does not exit, so systemd never gets involved. Don't disable - `auto_reconnect` or set `max_reconnect_attempts` under systemd — that just - replaces the client's backoff with unit restarts, which re-bind listeners - and drop held proxy requests. + Reconnects that keep failing escalate every third attempt to rebuilding the + iroh endpoint from scratch — the in-process equivalent of a unit restart, + covering wedges (a dead relay link, stale path state) that only a fresh + endpoint repairs. The process does not exit, so systemd never gets involved. + Don't disable `auto_reconnect` or set `max_reconnect_attempts` under + systemd — that just replaces the client's backoff with unit restarts, which + re-bind listeners and drop held proxy requests. - The client **exits nonzero** when the *first* connection fails (server down, network not up yet at boot) and on permanent auth/config errors. `Restart=on-failure` + `RestartSec` covers the boot-time window where the