Back off endpoint rebuilds during relay outages

AGENTS.md

@@ -0,0 +1,10 @@
+- strict no backward compatibility
+- run cargo clippy and test after rust code changes
+- to run the CI workflow's clippy + test steps on all three host platforms against the working tree, use `ci/all.sh` — see `docs/local-ci.md`. Worth doing before a release, or after touching platform-gated code (`flextunnel-desktop`'s macOS/Windows backends), which Linux-only checks never compile. If host is Linux, run linux ci on the same host.
+- no cargo fmt
+- no cargo test for flextunnel-desktop for linux because it is not available for linux
+- always use uv to run python scripts if needed
+- clients and server are expected to be trusted and error detections, for example, duplicate id detections are meant for preventing accidental misconfigurations such as running two clients or servers with the same id.
+- the desktop client (`flextunnel-desktop`) normally stores its config in the system keychain; set `FLEXTUNNEL_DEV_CONFIG=1` (or a file path) to store it as plaintext JSON instead, avoiding the macOS keychain access prompt on every unsigned rebuild. Development only — never set it for a real install (the auth secret key is stored unencrypted).
+- after rust changes that affect iOS (flextunnel-core or flextunnel-ffi, including the FFI config schema and `ios/flextunnel.h`), run `./build-ios.sh release` to rebuild `libflextunnel.xcframework` into `dist/ios/` (this script no longer writes into `../flextunnel-ios`). The iOS app links via its own Swift package (`../flextunnel-ios/Packages/Flextunnel`), which **defaults to the pinned GitHub release**, so it won't see local changes unless you build the app with `FLEXTUNNEL_LOCAL_XCFRAMEWORK=1` — that links this fresh `dist/ios` build through a committed symlink (set it for both `xcodegen generate` and `xcodebuild`, then clean-rebuild). This is an **extra step only needed when actively working on the iOS app side by side** (and only possible on macOS with Xcode + the iOS Rust targets); otherwise just skip it.
+- the iroh transport layer shared with tunnel-rs and ezvpn — relays and address lookup, the per-relay startup probe, relay auth tokens, relay self-hosting — is documented once in https://github.com/flexaccessdev/iroh-common-architecture. Do not duplicate it in this repo; update it there and link to it.

CLAUDE.md

@@ -1,9 +1 @@
-- strict no backward compatibility
-- run cargo clippy and test after rust code changes
-- to run the CI workflow's clippy + test steps on all three host platforms against the working tree, use `ci/all.sh` — see `docs/local-ci.md`. Worth doing before a release, or after touching platform-gated code (`flextunnel-desktop`'s macOS/Windows backends), which Linux-only checks never compile. If host is Linux, run linux ci on the same host.
-- no cargo fmt
-- always use uv to run python scripts if needed
-- clients and server are expected to be trusted and error detections, for example, duplicate id detections are meant for preventing accidental misconfigurations such as running two clients or servers with the same id.
-- the desktop client (`flextunnel-desktop`) normally stores its config in the system keychain; set `FLEXTUNNEL_DEV_CONFIG=1` (or a file path) to store it as plaintext JSON instead, avoiding the macOS keychain access prompt on every unsigned rebuild. Development only — never set it for a real install (the auth secret key is stored unencrypted).
-- after rust changes that affect iOS (flextunnel-core or flextunnel-ffi, including the FFI config schema and `ios/flextunnel.h`), run `./build-ios.sh release` to rebuild `libflextunnel.xcframework` into `dist/ios/` (this script no longer writes into `../flextunnel-ios`). The iOS app links via its own Swift package (`../flextunnel-ios/Packages/Flextunnel`), which **defaults to the pinned GitHub release**, so it won't see local changes unless you build the app with `FLEXTUNNEL_LOCAL_XCFRAMEWORK=1` — that links this fresh `dist/ios` build through a committed symlink (set it for both `xcodegen generate` and `xcodebuild`, then clean-rebuild). This is an **extra step only needed when actively working on the iOS app side by side** (and only possible on macOS with Xcode + the iOS Rust targets); otherwise just skip it.
-- the iroh transport layer shared with tunnel-rs and ezvpn — relays and address lookup, the per-relay startup probe, relay auth tokens, relay self-hosting — is documented once in https://github.com/flexaccessdev/iroh-common-architecture. Do not duplicate it in this repo; update it there and link to it.
+@AGENTS.md

Cargo.toml

@@ -10,7 +10,7 @@ default-members = [
 ]
 
 [workspace.package]
-version = "0.0.71"
+version = "0.0.72"
 edition = "2024"
 description = "SOCKS5/HTTP-proxy-over-QUIC split tunnel via iroh P2P"
 

crates/flextunnel-cli/src/main.rs

@@ -36,7 +36,7 @@ use flextunnel_core::transport::endpoint::{
     EndpointAllowlists, RelayConfig, create_server_endpoint, secret_to_endpoint_id,
     server_rebuild_factory,
 };
-use flextunnel_core::transport::relay_watchdog;
+use flextunnel_core::transport::relay_watchdog::{self, RelayOutage};
 use flextunnel_core::{auth, config, secret};
 
 #[derive(Parser)]
@@ -623,6 +623,22 @@ const SHUTDOWN_CLOSE_TIMEOUT: Duration = Duration::from_secs(5);
 /// short — there is nothing to lose by trying again soon.
 const REBUILD_RETRY: Duration = Duration::from_secs(30);
 
+/// Cap on the watchdog's rebuild deadline once consecutive rebuilt endpoints
+/// keep failing to register on any home relay.
+const REBUILD_DEADLINE_MAX: Duration = Duration::from_secs(30 * 60);
+
+/// The watchdog's rebuild deadline for the next serve pass, given how many
+/// endpoints in a row never registered on a home relay: the usual
+/// [`relay_watchdog::RELAY_OUTAGE_REBUILD`] after an endpoint that did
+/// register, doubling per unregistered endpoint up to [`REBUILD_DEADLINE_MAX`]
+/// (180s, 6m, 12m, 24m, 30m). Rebuilding while the relay itself is down
+/// gains nothing and drops every LAN client, so it is done less and less
+/// often; a relay that comes back resets the escalation.
+fn rebuild_deadline(unregistered_endpoints: u32) -> Duration {
+    let factor = 1u32 << unregistered_endpoints.min(4);
+    (relay_watchdog::RELAY_OUTAGE_REBUILD * factor).min(REBUILD_DEADLINE_MAX)
+}
+
 /// Build the ephemeral `ServerConfig` for `server start --quick`: a full-tunnel
 /// routed set (`routed_domains = ["*"]`, `routed_cidrs = ["0.0.0.0/0", "::/0"]`)
 /// plus a freshly generated in-memory identity, returned *alongside* the config —
@@ -885,8 +901,8 @@ async fn run_server(
     enum Pass {
         /// The server is done (clean or failed): close the endpoint and return.
         Exit(Result<()>),
-        /// The relay watchdog gave up on the endpoint after an outage this long.
-        Rebuild(Duration),
+        /// The relay watchdog gave up on the endpoint.
+        Rebuild(RelayOutage),
     }
 
     // Serve loop. A pass serves on the current endpoint until the server ends,
@@ -897,13 +913,22 @@ async fn run_server(
     // one with the same identity, and serve again. The `ProxyServer` (its
     // registries, blocklist, status state) carries over; the old endpoint's
     // connections and bridge tasks end with it.
+    //
+    // A rebuild only helps when iroh's relay bookkeeping went stale. When the
+    // relay itself is unreachable the fresh endpoint never registers either,
+    // and rebuilding it again every few minutes would keep dropping the LAN
+    // clients that still work. So consecutive endpoints that never saw a home
+    // relay lengthen the watchdog's deadline (`rebuild_deadline`); one that
+    // did register resets the escalation.
     let mut endpoint = endpoint;
+    let mut unregistered_endpoints: u32 = 0;
     let res = loop {
         let pass = {
             let run = Arc::clone(&server).run(&endpoint);
+            let deadline = rebuild_deadline(unregistered_endpoints);
             let outage = async {
                 if relay_watchdog_armed {
-                    relay_watchdog::watch_home_relay(&endpoint).await
+                    relay_watchdog::watch_home_relay(&endpoint, deadline).await
                 } else {
                     std::future::pending().await
                 }
@@ -925,12 +950,21 @@ async fn run_server(
             Pass::Rebuild(outage) => outage,
         };
 
+        unregistered_endpoints = if outage.relay_seen { 0 } else { unregistered_endpoints + 1 };
         log::error!(
             "No connected home relay for {:.0}s despite a network re-check; rebuilding the \
              endpoint from scratch (server id stays {})",
-            outage.as_secs_f64(),
+            outage.duration.as_secs_f64(),
             endpoint.id()
         );
+        if unregistered_endpoints > 0 {
+            log::error!(
+                "{unregistered_endpoints} endpoint(s) in a row never registered on any home \
+                 relay; the relay itself is probably unreachable. If the rebuilt endpoint does \
+                 not register either, the next rebuild waits {}s",
+                rebuild_deadline(unregistered_endpoints).as_secs()
+            );
+        }
         close_endpoint_or_exit(&endpoint).await;
         endpoint = loop {
             match rebuild().await {
@@ -991,6 +1025,17 @@ mod tests {
     use super::*;
     use std::collections::HashMap;
 
+    #[test]
+    fn rebuild_deadline_doubles_per_unregistered_endpoint_up_to_the_cap() {
+        let base = relay_watchdog::RELAY_OUTAGE_REBUILD;
+        assert_eq!(rebuild_deadline(0), base);
+        assert_eq!(rebuild_deadline(1), base * 2);
+        assert_eq!(rebuild_deadline(2), base * 4);
+        assert_eq!(rebuild_deadline(3), base * 8);
+        assert_eq!(rebuild_deadline(4), REBUILD_DEADLINE_MAX);
+        assert_eq!(rebuild_deadline(50), REBUILD_DEADLINE_MAX);
+    }
+
     fn forwarder(suffix: &str) -> DnsForwarder {
         let mut m = HashMap::new();
         m.insert(suffix.to_string(), vec!["10.0.0.53".to_string()]);

crates/flextunnel-core/src/transport/endpoint.rs

@@ -473,7 +473,9 @@ async fn bind_server_endpoint(
 ///   recovery through the one relay that still answers.
 /// - **The online wait is tolerated failing.** A fresh endpoint is no worse
 ///   than the wedged one it replaces — LAN clients can still find it over
-///   mDNS — and the watchdog trips again if the relays stay unreachable.
+///   mDNS — and the watchdog trips again if the relays stay unreachable
+///   (with a lengthening deadline, so a dead relay does not churn the endpoint
+///   every few minutes; see `run_server`).
 pub fn server_rebuild_factory(
     relay_config: RelayConfig,
     secret: SecretKey,