From 016a7da0fff0dbd373929b767ec375bcac4ebefa Mon Sep 17 00:00:00 2001 From: Matt Hill <9935159+MattDHill@users.noreply.github.com> Date: Mon, 27 Jul 2026 13:56:14 -0600 Subject: [PATCH] =?UTF-8?q?fix:=20clear=20stale=20postmaster.pid=20before?= =?UTF-8?q?=20starting=20postgres;=2033.0.6:2=20=E2=86=92=2033.0.6:3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PostgreSQL writes postmaster.pid into PGDATA while running and removes it on a clean exit. A stop that never completes leaves it behind, and on the next start Postgres aborts with FATAL: lock file "postmaster.pid" already exists if the PID recorded there is alive. Every chain build gets a fresh PID namespace with a fresh, low PID assignment, so that PID is quite likely to be live and owned by an unrelated process — the guard misfires and the daemon never comes up. Nothing surfaces when it happens: the postgres ready check maps any non-zero pg_isready to 'loading' with display: null, and Daemon.runLoop restarts the process forever on a backoff. In setupMain that shows as a service stuck starting. In the update-time upgrade chain added in 33.0.6:0 it shows as runUntilSuccess exhausting its 30-minute budget and failing init with "Timed out waiting for postgres,upgrade,__RUN_UNTIL_SUCCESS", which then rolls the update back — so the update fails identically on every retry. getBaseDaemons is the only place in the stack that starts Postgres without this prelude. The SDK's Backups.withPgDump does it before each pg_ctl start (lib/backup/Backups.ts), and the 0.3.5x migration's relocatePostgres does it too — the daemon path was simply missed. Removal is unconditional, which is safe because nothing else can hold the data directory: backups only run once the service is stopped, both init paths run with the service stopped, and the chain reconciler fully terms an entry before starting its replacement. Runs as root so ownership can never block the unlink and wedge the chain on the new oneshot. Placing it in getBaseDaemons covers setupMain, install init and update init in one place. --- README.md | 4 +++- startos/utils.ts | 14 +++++++++++++- startos/versions/current.ts | 7 ++++++- 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 0cdba3b..82f8e48 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,9 @@ This package runs **four containers** as subcontainers: Architectures: x86_64, aarch64. -**Startup order:** A `chown` one-shot runs first alongside `postgres` and `valkey`. The `nextcloud` container waits until all three are ready before starting. The `cron` container waits for `nextcloud` to be ready. Version upgrades run earlier, during init (see [Installation and First-Run Flow](#installation-and-first-run-flow)); after `nextcloud` is ready, a `finish-upgrade` one-shot completes any interrupted upstream upgrade as a fallback (see [Health Checks](#health-checks)), and the `long-running-tasks` one-shot runs after it. +**Startup order:** Two one-shots run first — `chown` and `pg-recover`. `postgres` waits on `pg-recover`; `valkey` has no prerequisites. The `nextcloud` container waits until `chown`, `postgres` and `valkey` are all ready before starting. The `cron` container waits for `nextcloud` to be ready. Version upgrades run earlier, during init (see [Installation and First-Run Flow](#installation-and-first-run-flow)); after `nextcloud` is ready, a `finish-upgrade` one-shot completes any interrupted upstream upgrade as a fallback (see [Health Checks](#health-checks)), and the `long-running-tasks` one-shot runs after it. + +**Unclean shutdown recovery (`pg-recover`):** PostgreSQL writes `postmaster.pid` into its data directory while running and removes it on a clean exit. A stop that does not complete — a power loss, a forced stop, or an update that fails and rolls back — can leave that file behind. On the next start PostgreSQL reads the PID it names and aborts with `FATAL: lock file "postmaster.pid" already exists` if that PID is alive. Each chain build runs in a fresh PID namespace with a fresh, low PID assignment, so the recorded PID is quite likely to be live and to belong to some unrelated process — the guard misfires. The daemon then crash-loops indefinitely, since `pg_isready` reports `loading` rather than a failure. The `pg-recover` one-shot removes the stale file before `postgres` starts, so the database proceeds to normal WAL crash recovery instead. Removal is unconditional and safe because nothing else can hold the data directory: init and backups both run with the service stopped, and the chain reconciler terminates a daemon before starting its replacement. The SDK's own `Backups.withPgDump` does the same before each `pg_ctl start`. **ffmpeg:** The nextcloud image is built locally (extends `nextcloud:-apache`) to install `ffmpeg`, which Nextcloud's preview providers shell out to for video thumbnails. diff --git a/startos/utils.ts b/startos/utils.ts index 7e604de..0adb49f 100644 --- a/startos/utils.ts +++ b/startos/utils.ts @@ -138,6 +138,18 @@ export function getBaseDaemons( }, requires: [], }) + .addOneshot('pg-recover', { + subcontainer: postgresSub, + exec: { + // An unclean stop strands postmaster.pid, and Postgres aborts if the + // PID it names is alive — which, in a fresh PID namespace, is usually + // an unrelated process. As root, so ownership can never block the + // removal and wedge the chain on this oneshot. + command: ['rm', '-f', `${PGDATA}/postmaster.pid`], + user: 'root', + }, + requires: [], + }) .addDaemon('postgres', { subcontainer: postgresSub, exec: { @@ -167,7 +179,7 @@ export function getBaseDaemons( } }, }, - requires: [], + requires: ['pg-recover'], }) .addDaemon('valkey', { subcontainer: valkeySub, diff --git a/startos/versions/current.ts b/startos/versions/current.ts index 548add3..1c85246 100644 --- a/startos/versions/current.ts +++ b/startos/versions/current.ts @@ -202,7 +202,7 @@ const migrateNextcloud = async (effects: T.Effects) => { } export const current = VersionInfo.of({ - version: '33.0.6:2', + version: '33.0.6:3', releaseNotes: { en_US: `Adds File Browser External Storage integration and repackages Nextcloud on start-sdk 2.0 (bundled image updated to Nextcloud 33.0.6 — upstream security and bug fixes). @@ -217,6 +217,7 @@ export const current = VersionInfo.of({ **Fixes** - Fixed a bug where background network changes on the server could put Nextcloud into a restart loop. +- Fixed a bug where PostgreSQL could refuse to start after an unclean shutdown — a power loss, a forced stop, or a failed update — leaving Nextcloud stuck on "starting" or an update failing with a timeout. A stale database lock file is now cleared before PostgreSQL starts. Internal updates (start-sdk 2.0). @@ -234,6 +235,7 @@ Full changelog: https://github.com/nextcloud-releases/server/releases/tag/v33.0. **Correcciones** - Corregido un error por el que cambios de red en segundo plano en el servidor podían poner Nextcloud en un bucle de reinicios. +- Corregido un error por el que PostgreSQL podía negarse a arrancar tras un apagado no limpio —un corte de energía, una parada forzada o una actualización fallida—, dejando Nextcloud atascado en «iniciando» o provocando que una actualización fallara por tiempo de espera agotado. Ahora se elimina el archivo de bloqueo obsoleto de la base de datos antes de iniciar PostgreSQL. Actualizaciones internas (start-sdk 2.0). @@ -251,6 +253,7 @@ Registro de cambios completo: https://github.com/nextcloud-releases/server/relea **Fehlerkorrekturen** - Ein Fehler wurde behoben, durch den Netzwerkänderungen im Hintergrund auf dem Server Nextcloud in eine Neustart-Schleife versetzen konnten. +- Ein Fehler wurde behoben, durch den PostgreSQL nach einem unsauberen Herunterfahren — einem Stromausfall, einem erzwungenen Stopp oder einer fehlgeschlagenen Aktualisierung — den Start verweigern konnte, sodass Nextcloud im Zustand „wird gestartet" hängen blieb oder eine Aktualisierung mit einer Zeitüberschreitung fehlschlug. Eine veraltete Sperrdatei der Datenbank wird jetzt vor dem Start von PostgreSQL entfernt. Interne Aktualisierungen (start-sdk 2.0). @@ -268,6 +271,7 @@ Vollständige Änderungsliste: https://github.com/nextcloud-releases/server/rele **Poprawki** - Naprawiono błąd, przez który zmiany sieci w tle na serwerze mogły wprowadzić Nextcloud w pętlę restartów. +- Naprawiono błąd, przez który PostgreSQL mógł odmówić uruchomienia po nieczystym zamknięciu — awarii zasilania, wymuszonym zatrzymaniu lub nieudanej aktualizacji — pozostawiając Nextcloud w stanie „uruchamianie" lub powodując niepowodzenie aktualizacji z powodu przekroczenia limitu czasu. Nieaktualny plik blokady bazy danych jest teraz usuwany przed uruchomieniem PostgreSQL. Aktualizacje wewnętrzne (start-sdk 2.0). @@ -285,6 +289,7 @@ Pełny dziennik zmian: https://github.com/nextcloud-releases/server/releases/tag **Correctifs** - Correction d'un bogue où des changements réseau en arrière-plan sur le serveur pouvaient placer Nextcloud dans une boucle de redémarrages. +- Correction d'un bogue où PostgreSQL pouvait refuser de démarrer après un arrêt brutal — une coupure de courant, un arrêt forcé ou une mise à jour échouée —, laissant Nextcloud bloqué sur « démarrage » ou faisant échouer une mise à jour par dépassement de délai. Un fichier de verrou de base de données obsolète est désormais supprimé avant le démarrage de PostgreSQL. Mises à jour internes (start-sdk 2.0).