From 62e71ac391ee4cdb6e5f65dfeda9e9e71c779f63 Mon Sep 17 00:00:00 2001
From: Mondo Diaz <mondo.m.diaz@gmail.com>
Date: Tue, 5 May 2026 09:13:42 -0500
Subject: [PATCH] retry failed server connections at 60s cadence

generate_server_connections now records failures on app_state, and the
main loop's wait phase re-runs it every 60s while any are outstanding.
When all servers reconnect, fire request_manual_sync so the dashboard
goes green and a sync runs immediately instead of waiting for the next
hourly cycle. Fixes Plex/Jellyfin red dots after Unraid reboots when
those services are still in startup maintenance at ferret boot time.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 src/connection.py |  3 +++
 src/main.py       | 31 ++++++++++++++++++++++++++++++-
 src/web/state.py  | 16 ++++++++++++++++
 3 files changed, 49 insertions(+), 1 deletion(-)

diff --git a/src/connection.py b/src/connection.py
index 3930e9e..3e65b1b 100644
--- a/src/connection.py
+++ b/src/connection.py
@@ -83,6 +83,7 @@ def jellyfin_emby_server_connection(
                     status="disconnected",
                     version=""
                 )
+                app_state.increment_connection_failures()
             logger.warning("Continuing with remaining servers...")
 
     return servers
@@ -155,6 +156,7 @@ def generate_server_connections(env) -> list[Plex | Jellyfin | Emby]:
                         status="disconnected",
                         version=""
                     )
+                    app_state.increment_connection_failures()
                 logger.warning("Continuing with remaining servers...")
 
     if plex_username_str and plex_password_str and plex_servername_str:
@@ -208,6 +210,7 @@ def generate_server_connections(env) -> list[Plex | Jellyfin | Emby]:
                         status="disconnected",
                         version=""
                     )
+                    app_state.increment_connection_failures()
                 logger.warning("Continuing with remaining servers...")
 
     jellyfin_baseurl = get_env_value(env, "JELLYFIN_BASEURL", None)
diff --git a/src/main.py b/src/main.py
index 8b1d743..67f4055 100644
--- a/src/main.py
+++ b/src/main.py
@@ -435,13 +435,42 @@ def main() -> None:
 
             # Check for manual sync requests from web dashboard
             if web_dashboard_enabled:
-                # Sleep in small intervals to check for manual sync requests
+                # Sleep in small intervals to check for manual sync requests.
+                # If any servers failed to connect this cycle (e.g. they were
+                # still in startup maintenance after an Unraid reboot), retry
+                # connections every CONNECTION_RETRY_INTERVAL seconds and
+                # trigger an immediate sync if they all come back.
+                CONNECTION_RETRY_INTERVAL = 60
                 elapsed = 0
+                last_retry_at = 0
                 while elapsed < sleep_duration:
                     if app_state.manual_sync_requested:
                         logger.info("Web: Manual sync request detected, starting sync now")
                         app_state.clear_manual_sync_request()
                         break
+
+                    if (
+                        app_state.get_connection_failures() > 0
+                        and (elapsed - last_retry_at) >= CONNECTION_RETRY_INTERVAL
+                    ):
+                        last_retry_at = elapsed
+                        prior_failures = app_state.get_connection_failures()
+                        logger.info(
+                            f"Retrying {prior_failures} failed server connection(s)..."
+                        )
+                        try:
+                            generate_server_connections(env)
+                        except Exception as e:
+                            logger.error(f"Reconnect attempt failed: {e}")
+                        if (
+                            app_state.get_connection_failures() == 0
+                            and prior_failures > 0
+                        ):
+                            logger.info(
+                                "All servers reconnected; triggering immediate sync"
+                            )
+                            app_state.request_manual_sync()
+
                     sleep(1)
                     elapsed += 1
             else:
diff --git a/src/web/state.py b/src/web/state.py
index 95f0d48..2e9568f 100644
--- a/src/web/state.py
+++ b/src/web/state.py
@@ -26,6 +26,12 @@ def __init__(self):
         # Server status
         self.servers = {}  # {server_name: {type, status, version}}
 
+        # Number of servers that failed to connect during the most recent
+        # connection cycle. Used by the main loop to decide whether to retry
+        # connections at a faster cadence (e.g. after Unraid reboot, when
+        # Plex/Jellyfin may still be in startup maintenance).
+        self.connection_failures = 0
+
         # Statistics
         self.total_items_synced = 0
         self.total_users = 0
@@ -83,6 +89,16 @@ def reset_servers(self):
         """
         with self._lock:
             self.servers = {}
+            self.connection_failures = 0
+
+    def increment_connection_failures(self):
+        """Record that a server failed to connect during the current cycle."""
+        with self._lock:
+            self.connection_failures += 1
+
+    def get_connection_failures(self) -> int:
+        with self._lock:
+            return self.connection_failures
 
     def update_stats(self, items_synced: int = 0, users: int = 0):
         """Update sync statistics."""